01:40:49.496 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
01:40:49.509 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task5IFP5y/test-tmp
01:40:49.511 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
01:40:49.515 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task5IFP5y/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
01:40:49.764 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
01:40:49.888 [DEBUG - main] (MiniKuduCluster.java:195) Response:
01:40:49.889 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
01:40:49.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:49.892660 374 mini_oidc.cc:152] Starting JWKS server
01:40:49.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:49.897213 374 webserver.cc:460] Webserver started at http://127.0.0.1:38831/ using document root <none> and password file <none>
01:40:49.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:49.897769 374 mini_oidc.cc:189] Starting OIDC Discovery server
01:40:49.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:49.899278 374 webserver.cc:460] Webserver started at http://127.0.0.1:40573/ using document root <none> and password file <none>
01:40:49.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T01:40:49Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
01:40:49.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T01:40:49Z Disabled control of system clock
01:40:49.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:49.947516 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-task5IFP5y/build/tsan/bin/kudu
01:40:49.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5IFP5y/build/tsan/bin/kudu
01:40:49.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/master-0/wal
01:40:49.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/master-0/data
01:40:49.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:40:49.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:40:49.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:40:49.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:40:49.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:40:49.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:40:49.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/master-0/logs
01:40:49.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/master-0/data/info.pb
01:40:49.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:40:49.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:40:49.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:40:49.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:40:49.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:40:49.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:40:49.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:40:49.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:40:49.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:40:49.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:40:49.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:40:49.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:40:49.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.190:34177
01:40:49.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.190
01:40:49.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:40:49.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:37611
01:40:49.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:40:49.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:40:49.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:40:49.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:40:49.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:34177,127.0.93.189:33795,127.0.93.188:40257
01:40:49.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:40:49.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38831/jwks.json with env {}
01:40:50.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:50.237733 386 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:40:50.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:50.238255 386 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:40:50.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:50.238791 386 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:40:50.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:50.276417 386 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:40:50.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:50.276675 386 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:40:50.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:50.276860 386 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:40:50.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:50.277112 386 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:40:50.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:50.277371 386 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:40:50.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:50.277567 386 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38831/jwks.json
01:40:50.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:50.310693 386 master_runner.cc:386] Master server non-default flags:
01:40:50.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:40:50.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:37611
01:40:50.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:40:50.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:40:50.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/master-0/data
01:40:50.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/master-0/wal
01:40:50.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:40:50.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:34177,127.0.93.189:33795,127.0.93.188:40257
01:40:50.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:40:50.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:40:50.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:40:50.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.190:34177
01:40:50.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:40:50.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:40:50.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:40:50.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38831/jwks.json
01:40:50.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:40:50.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:40:50.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/master-0/data/info.pb
01:40:50.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.190
01:40:50.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:40:50.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:40:50.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:40:50.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:40:50.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:40:50.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:40:50.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/master-0/logs
01:40:50.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:40:50.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:40:50.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:40:50.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:40:50.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:40:50.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:40:50.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:40:50.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 00:23:04 UTC on ccycloud.quasar-noncbr.root.comops.site
01:40:50.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4678933
01:40:50.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:40:50.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:50.311987 386 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:40:50.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:50.313556 386 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:40:50.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:50.332760 392 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:40:51.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:50.334133 393 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:40:51.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:50.332753 395 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:40:51.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:51.531325 394 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1197 milliseconds
01:40:51.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:51.531423 386 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
01:40:51.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:51.532789 386 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:40:51.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:51.535960 386 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:40:51.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:51.537376 386 hybrid_clock.cc:648] HybridClock initialized: now 1790646051537347 us; error 49 us; skew 500 ppm
01:40:51.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:51.538106 386 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:40:51.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:51.544173 386 webserver.cc:460] Webserver started at http://127.0.93.190:46333/ using document root <none> and password file <none>
01:40:51.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:51.544992 386 fs_manager.cc:359] Metadata directory not provided
01:40:51.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:51.545166 386 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:40:51.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:51.545531 386 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:40:51.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:51.550136 386 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/master-0/data/instance:
01:40:51.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6d464f9db55d4ea39860d05aa9275f58"
01:40:51.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:40:51 on dist-test-slave-rw6r"
01:40:51.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:51.551345 386 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/master-0/wal/instance:
01:40:51.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6d464f9db55d4ea39860d05aa9275f58"
01:40:51.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:40:51 on dist-test-slave-rw6r"
01:40:51.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:51.558537 386 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.000s
01:40:51.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:51.563457 402 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:40:51.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:51.564708 386 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
01:40:51.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:51.565025 386 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/master-0/data,/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/master-0/wal
01:40:51.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6d464f9db55d4ea39860d05aa9275f58"
01:40:51.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:40:51 on dist-test-slave-rw6r"
01:40:51.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:51.565394 386 fs_report.cc:383] FS layout report
01:40:51.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:40:51.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/master-0/wal
01:40:51.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/master-0/wal
01:40:51.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/master-0/data/data
01:40:51.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:40:51.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:40:51.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:40:51.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:40:51.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:40:51.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:40:51.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:40:51.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:40:51.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:40:51.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:40:51.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:40:51.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:51.608364 386 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:40:51.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:51.609815 386 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:40:51.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:51.610301 386 kserver.cc:163] Server-wide thread pool size limit: 3276
01:40:51.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:51.681610 386 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.190:34177
01:40:51.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:51.681704 453 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.190:34177 every 8 connection(s)
01:40:51.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:51.684432 386 server_base.cc:1166] Dumped server information to /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/master-0/data/info.pb
01:40:51.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:51.689571 454 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:40:51.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:51.694936 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-task5IFP5y/build/tsan/bin/kudu as pid 386
01:40:51.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:51.695415 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/master-0/wal/instance
01:40:51.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:51.698853 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-task5IFP5y/build/tsan/bin/kudu
01:40:51.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5IFP5y/build/tsan/bin/kudu
01:40:51.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/master-1/wal
01:40:51.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/master-1/data
01:40:51.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:40:51.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:40:51.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:40:51.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:40:51.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:40:51.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:40:51.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/master-1/logs
01:40:51.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/master-1/data/info.pb
01:40:51.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:40:51.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:40:51.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:40:51.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:40:51.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:40:51.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:40:51.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:40:51.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:40:51.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:40:51.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:40:51.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:40:51.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:40:51.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.189:33795
01:40:51.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.189
01:40:51.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:40:51.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:37611
01:40:51.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:40:51.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:40:51.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:40:51.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:40:51.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:34177,127.0.93.189:33795,127.0.93.188:40257
01:40:51.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:40:51.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38831/jwks.json with env {}
01:40:51.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:51.701275 454 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.190" port: 34177 } has no permanent_uuid. Determining permanent_uuid...
01:40:51.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:51.729799 454 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33795 } has no permanent_uuid. Determining permanent_uuid...
01:40:51.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:51.732175 405 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.93.189:33795: connect: Connection refused (error 111)
01:40:51.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:51.736382 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:33795: Network error: Client connection negotiation failed: client connection to 127.0.93.189:33795: connect: Connection refused (error 111)
01:40:51.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:51.785903 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33795 } attempt: 1
01:40:51.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:51.789932 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:33795: Network error: Client connection negotiation failed: client connection to 127.0.93.189:33795: connect: Connection refused (error 111)
01:40:51.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:51.858434 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33795 } attempt: 2
01:40:51.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:51.862931 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:33795: Network error: Client connection negotiation failed: client connection to 127.0.93.189:33795: connect: Connection refused (error 111)
01:40:51.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:51.954372 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33795 } attempt: 3
01:40:51.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:51.958455 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:33795: Network error: Client connection negotiation failed: client connection to 127.0.93.189:33795: connect: Connection refused (error 111)
01:40:52.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:52.002441 456 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:40:52.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:52.003111 456 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:40:52.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:52.003784 456 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:40:52.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:52.033556 456 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:40:52.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:52.033926 456 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:40:52.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:52.034255 456 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:40:52.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:52.034597 456 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:40:52.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:52.034955 456 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:40:52.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:52.035202 456 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38831/jwks.json
01:40:52.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:52.067935 456 master_runner.cc:386] Master server non-default flags:
01:40:52.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:40:52.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:37611
01:40:52.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:40:52.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:40:52.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/master-1/data
01:40:52.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/master-1/wal
01:40:52.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:40:52.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:34177,127.0.93.189:33795,127.0.93.188:40257
01:40:52.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:40:52.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:40:52.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:40:52.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.189:33795
01:40:52.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:40:52.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:40:52.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:40:52.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38831/jwks.json
01:40:52.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:40:52.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:40:52.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/master-1/data/info.pb
01:40:52.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.189
01:40:52.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:40:52.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:40:52.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:40:52.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:40:52.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:40:52.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:40:52.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/master-1/logs
01:40:52.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:40:52.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:40:52.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:40:52.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:40:52.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:40:52.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:40:52.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:40:52.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 00:23:04 UTC on ccycloud.quasar-noncbr.root.comops.site
01:40:52.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4678933
01:40:52.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:40:52.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:52.069396 456 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:40:52.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:52.071496 456 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:40:52.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:52.089514 468 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:40:52.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:52.101800 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33795 } attempt: 4
01:40:52.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:52.105809 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:33795: Network error: Client connection negotiation failed: client connection to 127.0.93.189:33795: connect: Connection refused (error 111)
01:40:52.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:52.405910 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33795 } attempt: 5
01:40:52.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:52.410559 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:33795: Network error: Client connection negotiation failed: client connection to 127.0.93.189:33795: connect: Connection refused (error 111)
01:40:52.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:52.958184 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33795 } attempt: 6
01:40:52.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:52.965320 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:33795: Network error: Client connection negotiation failed: client connection to 127.0.93.189:33795: connect: Connection refused (error 111)
01:40:53.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:52.089566 466 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:40:53.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:52.090499 465 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:40:53.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:53.275033 456 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
01:40:53.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:53.275517 467 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
01:40:53.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:53.279033 456 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:40:53.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:53.283074 456 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:40:53.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:53.284500 456 hybrid_clock.cc:648] HybridClock initialized: now 1790646053284476 us; error 58 us; skew 500 ppm
01:40:53.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:53.285509 456 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:40:53.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:53.297984 456 webserver.cc:460] Webserver started at http://127.0.93.189:43287/ using document root <none> and password file <none>
01:40:53.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:53.298877 456 fs_manager.cc:359] Metadata directory not provided
01:40:53.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:53.299053 456 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:40:53.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:53.299422 456 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:40:53.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:53.304330 456 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/master-1/data/instance:
01:40:53.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "060ada4773214290b7ff6c513dd42334"
01:40:53.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:40:53 on dist-test-slave-rw6r"
01:40:53.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:53.305619 456 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/master-1/wal/instance:
01:40:53.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "060ada4773214290b7ff6c513dd42334"
01:40:53.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:40:53 on dist-test-slave-rw6r"
01:40:53.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:53.312942 456 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.000s sys 0.008s
01:40:53.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:53.318362 477 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:40:53.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:53.319401 456 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.000s
01:40:53.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:53.319679 456 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/master-1/data,/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/master-1/wal
01:40:53.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "060ada4773214290b7ff6c513dd42334"
01:40:53.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:40:53 on dist-test-slave-rw6r"
01:40:53.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:53.320044 456 fs_report.cc:383] FS layout report
01:40:53.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:40:53.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/master-1/wal
01:40:53.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/master-1/wal
01:40:53.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/master-1/data/data
01:40:53.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:40:53.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:40:53.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:40:53.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:40:53.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:40:53.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:40:53.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:40:53.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:40:53.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:40:53.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:40:53.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:40:53.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:53.355149 456 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:40:53.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:53.356582 456 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:40:53.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:53.357367 456 kserver.cc:163] Server-wide thread pool size limit: 3276
01:40:53.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:53.428579 456 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.189:33795
01:40:53.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:53.429068 528 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.189:33795 every 8 connection(s)
01:40:53.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:53.431337 456 server_base.cc:1166] Dumped server information to /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/master-1/data/info.pb
01:40:53.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:53.436472 529 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:40:53.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:53.440830 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-task5IFP5y/build/tsan/bin/kudu as pid 456
01:40:53.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:53.441260 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/master-1/wal/instance
01:40:53.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:53.444691 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-task5IFP5y/build/tsan/bin/kudu
01:40:53.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5IFP5y/build/tsan/bin/kudu
01:40:53.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/master-2/wal
01:40:53.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/master-2/data
01:40:53.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:40:53.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:40:53.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:40:53.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:40:53.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:40:53.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:40:53.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/master-2/logs
01:40:53.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/master-2/data/info.pb
01:40:53.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:40:53.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:40:53.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:40:53.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:40:53.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:40:53.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:40:53.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:40:53.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:40:53.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:40:53.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:40:53.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:40:53.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:40:53.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.188:40257
01:40:53.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.188
01:40:53.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:40:53.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:37611
01:40:53.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:40:53.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:40:53.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:40:53.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:40:53.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:34177,127.0.93.189:33795,127.0.93.188:40257
01:40:53.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:40:53.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38831/jwks.json with env {}
01:40:53.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:53.451968 529 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.190" port: 34177 } has no permanent_uuid. Determining permanent_uuid...
01:40:53.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:53.475347 529 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33795 } has no permanent_uuid. Determining permanent_uuid...
01:40:53.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:53.492592 529 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40257 } has no permanent_uuid. Determining permanent_uuid...
01:40:53.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:53.494798 478 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.93.188:40257: connect: Connection refused (error 111)
01:40:53.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:53.499281 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:40257: Network error: Client connection negotiation failed: client connection to 127.0.93.188:40257: connect: Connection refused (error 111)
01:40:53.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:53.548717 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40257 } attempt: 1
01:40:53.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:53.552253 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:40257: Network error: Client connection negotiation failed: client connection to 127.0.93.188:40257: connect: Connection refused (error 111)
01:40:53.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:53.620601 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40257 } attempt: 2
01:40:53.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:53.624518 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:40257: Network error: Client connection negotiation failed: client connection to 127.0.93.188:40257: connect: Connection refused (error 111)
01:40:53.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:53.715878 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40257 } attempt: 3
01:40:53.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:53.719548 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:40257: Network error: Client connection negotiation failed: client connection to 127.0.93.188:40257: connect: Connection refused (error 111)
01:40:53.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:53.758463 531 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:40:53.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:53.759006 531 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:40:53.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:53.759492 531 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:40:53.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:53.788872 531 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:40:53.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:53.789129 531 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:40:53.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:53.789409 531 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:40:53.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:53.789716 531 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:40:53.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:53.789916 531 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:40:53.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:53.790042 531 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38831/jwks.json
01:40:53.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:53.824409 531 master_runner.cc:386] Master server non-default flags:
01:40:53.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:40:53.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:37611
01:40:53.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:40:53.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:40:53.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/master-2/data
01:40:53.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/master-2/wal
01:40:53.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:40:53.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:34177,127.0.93.189:33795,127.0.93.188:40257
01:40:53.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:40:53.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:40:53.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:40:53.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.188:40257
01:40:53.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:40:53.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:40:53.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:40:53.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38831/jwks.json
01:40:53.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:40:53.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:40:53.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/master-2/data/info.pb
01:40:53.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.188
01:40:53.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:40:53.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:40:53.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:40:53.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:40:53.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:40:53.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:40:53.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/master-2/logs
01:40:53.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:40:53.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:40:53.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:40:53.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:40:53.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:40:53.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:40:53.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:40:53.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 00:23:04 UTC on ccycloud.quasar-noncbr.root.comops.site
01:40:53.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4678933
01:40:53.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:40:53.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:53.825726 531 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:40:53.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:53.827661 531 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:40:53.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:53.848883 541 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:40:53.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:53.862931 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40257 } attempt: 4
01:40:53.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:53.867610 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:40257: Network error: Client connection negotiation failed: client connection to 127.0.93.188:40257: connect: Connection refused (error 111)
01:40:54.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:54.025879 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33795 } attempt: 7
01:40:54.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:54.038833 454 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40257 } has no permanent_uuid. Determining permanent_uuid...
01:40:54.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:54.044629 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:40257: Network error: Client connection negotiation failed: client connection to 127.0.93.188:40257: connect: Connection refused (error 111)
01:40:54.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:54.103253 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40257 } attempt: 1
01:40:54.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:54.107215 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:40257: Network error: Client connection negotiation failed: client connection to 127.0.93.188:40257: connect: Connection refused (error 111)
01:40:54.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:54.167096 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40257 } attempt: 5
01:40:54.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:54.171454 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:40257: Network error: Client connection negotiation failed: client connection to 127.0.93.188:40257: connect: Connection refused (error 111)
01:40:54.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:54.188804 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40257 } attempt: 2
01:40:54.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:54.194104 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:40257: Network error: Client connection negotiation failed: client connection to 127.0.93.188:40257: connect: Connection refused (error 111)
01:40:54.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:54.279718 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40257 } attempt: 3
01:40:54.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:54.284194 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:40257: Network error: Client connection negotiation failed: client connection to 127.0.93.188:40257: connect: Connection refused (error 111)
01:40:54.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:54.424588 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40257 } attempt: 4
01:40:54.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:54.429836 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:40257: Network error: Client connection negotiation failed: client connection to 127.0.93.188:40257: connect: Connection refused (error 111)
01:40:54.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:54.713239 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40257 } attempt: 5
01:40:54.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:54.716722 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:40257: Network error: Client connection negotiation failed: client connection to 127.0.93.188:40257: connect: Connection refused (error 111)
01:40:54.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:54.718868 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40257 } attempt: 6
01:40:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:54.726339 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:40257: Network error: Client connection negotiation failed: client connection to 127.0.93.188:40257: connect: Connection refused (error 111)
01:40:55.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:53.849068 542 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:40:55.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:53.850483 544 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:40:55.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:55.045724 543 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
01:40:55.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.045806 531 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
01:40:55.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.050112 531 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:40:55.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.052677 531 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:40:55.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.054049 531 hybrid_clock.cc:648] HybridClock initialized: now 1790646055054027 us; error 65 us; skew 500 ppm
01:40:55.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.054957 531 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:40:55.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.062995 531 webserver.cc:460] Webserver started at http://127.0.93.188:38599/ using document root <none> and password file <none>
01:40:55.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.064276 531 fs_manager.cc:359] Metadata directory not provided
01:40:55.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.064570 531 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:40:55.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.065114 531 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:40:55.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.072542 531 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/master-2/data/instance:
01:40:55.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6c40c560f2cc4dbca2d2f3ca0e169156"
01:40:55.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:40:55 on dist-test-slave-rw6r"
01:40:55.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.073913 531 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/master-2/wal/instance:
01:40:55.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6c40c560f2cc4dbca2d2f3ca0e169156"
01:40:55.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:40:55 on dist-test-slave-rw6r"
01:40:55.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.080914 531 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.007s sys 0.001s
01:40:55.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.086848 556 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:40:55.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.088408 531 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.002s sys 0.000s
01:40:55.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.088757 531 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/master-2/data,/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/master-2/wal
01:40:55.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6c40c560f2cc4dbca2d2f3ca0e169156"
01:40:55.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:40:55 on dist-test-slave-rw6r"
01:40:55.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.089154 531 fs_report.cc:383] FS layout report
01:40:55.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:40:55.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/master-2/wal
01:40:55.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/master-2/wal
01:40:55.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/master-2/data/data
01:40:55.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:40:55.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:40:55.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:40:55.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:40:55.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:40:55.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:40:55.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:40:55.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:40:55.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:40:55.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:40:55.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:40:55.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.130394 531 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:40:55.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.132019 531 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:40:55.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.132560 531 kserver.cc:163] Server-wide thread pool size limit: 3276
01:40:55.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.210779 531 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.188:40257
01:40:55.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.210848 607 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.188:40257 every 8 connection(s)
01:40:55.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.213675 531 server_base.cc:1166] Dumped server information to /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/master-2/data/info.pb
01:40:55.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.217841 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-task5IFP5y/build/tsan/bin/kudu as pid 531
01:40:55.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.218326 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/master-2/wal/instance
01:40:55.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.219822 608 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:40:55.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.236701 608 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.190" port: 34177 } has no permanent_uuid. Determining permanent_uuid...
01:40:55.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.262826 608 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33795 } has no permanent_uuid. Determining permanent_uuid...
01:40:55.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.269290 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40257 } attempt: 6
01:40:55.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.276708 608 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40257 } has no permanent_uuid. Determining permanent_uuid...
01:40:55.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.300904 454 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6d464f9db55d4ea39860d05aa9275f58: Bootstrap starting.
01:40:55.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.300937 608 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6c40c560f2cc4dbca2d2f3ca0e169156: Bootstrap starting.
01:40:55.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.308529 454 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 6d464f9db55d4ea39860d05aa9275f58: Neither blocks nor log segments found. Creating new log.
01:40:55.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.309100 608 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 6c40c560f2cc4dbca2d2f3ca0e169156: Neither blocks nor log segments found. Creating new log.
01:40:55.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.310292 454 log.cc:824] T 00000000000000000000000000000000 P 6d464f9db55d4ea39860d05aa9275f58: Log is configured to *not* fsync() on all Append() calls
01:40:55.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.311172 608 log.cc:824] T 00000000000000000000000000000000 P 6c40c560f2cc4dbca2d2f3ca0e169156: Log is configured to *not* fsync() on all Append() calls
01:40:55.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.315474 454 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6d464f9db55d4ea39860d05aa9275f58: No bootstrap required, opened a new log
01:40:55.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.316664 608 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6c40c560f2cc4dbca2d2f3ca0e169156: No bootstrap required, opened a new log
01:40:55.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.337982 454 raft_consensus.cc:348] T 00000000000000000000000000000000 P 6d464f9db55d4ea39860d05aa9275f58 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d464f9db55d4ea39860d05aa9275f58" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 34177 } } peers { permanent_uuid: "060ada4773214290b7ff6c513dd42334" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33795 } } peers { permanent_uuid: "6c40c560f2cc4dbca2d2f3ca0e169156" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40257 } }
01:40:55.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.339052 454 raft_consensus.cc:374] T 00000000000000000000000000000000 P 6d464f9db55d4ea39860d05aa9275f58 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
01:40:55.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.339370 454 raft_consensus.cc:729] T 00000000000000000000000000000000 P 6d464f9db55d4ea39860d05aa9275f58 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d464f9db55d4ea39860d05aa9275f58, State: Initialized, Role: FOLLOWER
01:40:55.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.339988 608 raft_consensus.cc:348] T 00000000000000000000000000000000 P 6c40c560f2cc4dbca2d2f3ca0e169156 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d464f9db55d4ea39860d05aa9275f58" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 34177 } } peers { permanent_uuid: "060ada4773214290b7ff6c513dd42334" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33795 } } peers { permanent_uuid: "6c40c560f2cc4dbca2d2f3ca0e169156" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40257 } }
01:40:55.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.340709 608 raft_consensus.cc:374] T 00000000000000000000000000000000 P 6c40c560f2cc4dbca2d2f3ca0e169156 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
01:40:55.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.340360 454 consensus_queue.cc:260] T 00000000000000000000000000000000 P 6d464f9db55d4ea39860d05aa9275f58 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d464f9db55d4ea39860d05aa9275f58" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 34177 } } peers { permanent_uuid: "060ada4773214290b7ff6c513dd42334" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33795 } } peers { permanent_uuid: "6c40c560f2cc4dbca2d2f3ca0e169156" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40257 } }
01:40:55.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.341012 608 raft_consensus.cc:729] T 00000000000000000000000000000000 P 6c40c560f2cc4dbca2d2f3ca0e169156 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6c40c560f2cc4dbca2d2f3ca0e169156, State: Initialized, Role: FOLLOWER
01:40:55.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.342130 608 consensus_queue.cc:260] T 00000000000000000000000000000000 P 6c40c560f2cc4dbca2d2f3ca0e169156 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d464f9db55d4ea39860d05aa9275f58" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 34177 } } peers { permanent_uuid: "060ada4773214290b7ff6c513dd42334" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33795 } } peers { permanent_uuid: "6c40c560f2cc4dbca2d2f3ca0e169156" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40257 } }
01:40:55.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.345224 620 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6d464f9db55d4ea39860d05aa9275f58 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d464f9db55d4ea39860d05aa9275f58" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 34177 } } peers { permanent_uuid: "060ada4773214290b7ff6c513dd42334" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33795 } } peers { permanent_uuid: "6c40c560f2cc4dbca2d2f3ca0e169156" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40257 } } }
01:40:55.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.346549 454 sys_catalog.cc:564] T 00000000000000000000000000000000 P 6d464f9db55d4ea39860d05aa9275f58 [sys.catalog]: configured and running, proceeding with master startup.
01:40:55.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.346990 620 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6d464f9db55d4ea39860d05aa9275f58 [sys.catalog]: This master's current role is: FOLLOWER
01:40:55.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.348941 621 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6c40c560f2cc4dbca2d2f3ca0e169156 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d464f9db55d4ea39860d05aa9275f58" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 34177 } } peers { permanent_uuid: "060ada4773214290b7ff6c513dd42334" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33795 } } peers { permanent_uuid: "6c40c560f2cc4dbca2d2f3ca0e169156" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40257 } } }
01:40:55.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.349792 621 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6c40c560f2cc4dbca2d2f3ca0e169156 [sys.catalog]: This master's current role is: FOLLOWER
01:40:55.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.350654 608 sys_catalog.cc:564] T 00000000000000000000000000000000 P 6c40c560f2cc4dbca2d2f3ca0e169156 [sys.catalog]: configured and running, proceeding with master startup.
01:40:55.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:55.369309 408 tablet.cc:2369] T 00000000000000000000000000000000 P 6d464f9db55d4ea39860d05aa9275f58: Can't schedule compaction. Clean time has not been advanced past its initial value.
01:40:55.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.381682 621 raft_consensus.cc:482] T 00000000000000000000000000000000 P 6c40c560f2cc4dbca2d2f3ca0e169156 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:40:55.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.382252 621 raft_consensus.cc:504] T 00000000000000000000000000000000 P 6c40c560f2cc4dbca2d2f3ca0e169156 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d464f9db55d4ea39860d05aa9275f58" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 34177 } } peers { permanent_uuid: "060ada4773214290b7ff6c513dd42334" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33795 } } peers { permanent_uuid: "6c40c560f2cc4dbca2d2f3ca0e169156" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40257 } }
01:40:55.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.386520 621 leader_election.cc:288] T 00000000000000000000000000000000 P 6c40c560f2cc4dbca2d2f3ca0e169156 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6d464f9db55d4ea39860d05aa9275f58 (127.0.93.190:34177), 060ada4773214290b7ff6c513dd42334 (127.0.93.189:33795)
01:40:55.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:55.389621 562 tablet.cc:2369] T 00000000000000000000000000000000 P 6c40c560f2cc4dbca2d2f3ca0e169156: Can't schedule compaction. Clean time has not been advanced past its initial value.
01:40:55.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.390169 429 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6c40c560f2cc4dbca2d2f3ca0e169156" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d464f9db55d4ea39860d05aa9275f58" is_pre_election: true
01:40:55.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.391005 429 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 6d464f9db55d4ea39860d05aa9275f58 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6c40c560f2cc4dbca2d2f3ca0e169156 in term 0.
01:40:55.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.404259 558 leader_election.cc:302] T 00000000000000000000000000000000 P 6c40c560f2cc4dbca2d2f3ca0e169156 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6c40c560f2cc4dbca2d2f3ca0e169156, 6d464f9db55d4ea39860d05aa9275f58; no voters:
01:40:55.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.409907 621 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 6c40c560f2cc4dbca2d2f3ca0e169156 [term 0 FOLLOWER]: Leader pre-election won for term 1
01:40:55.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.410315 621 raft_consensus.cc:482] T 00000000000000000000000000000000 P 6c40c560f2cc4dbca2d2f3ca0e169156 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
01:40:55.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.410672 621 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 6c40c560f2cc4dbca2d2f3ca0e169156 [term 0 FOLLOWER]: Advancing to term 1
01:40:55.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.441069 621 raft_consensus.cc:504] T 00000000000000000000000000000000 P 6c40c560f2cc4dbca2d2f3ca0e169156 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d464f9db55d4ea39860d05aa9275f58" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 34177 } } peers { permanent_uuid: "060ada4773214290b7ff6c513dd42334" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33795 } } peers { permanent_uuid: "6c40c560f2cc4dbca2d2f3ca0e169156" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40257 } }
01:40:55.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.458244 429 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6c40c560f2cc4dbca2d2f3ca0e169156" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d464f9db55d4ea39860d05aa9275f58"
01:40:55.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.459049 429 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 6d464f9db55d4ea39860d05aa9275f58 [term 0 FOLLOWER]: Advancing to term 1
01:40:55.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.471623 621 leader_election.cc:288] T 00000000000000000000000000000000 P 6c40c560f2cc4dbca2d2f3ca0e169156 [CANDIDATE]: Term 1 election: Requested vote from peers 6d464f9db55d4ea39860d05aa9275f58 (127.0.93.190:34177), 060ada4773214290b7ff6c513dd42334 (127.0.93.189:33795)
01:40:55.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:55.479640 643 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 6d464f9db55d4ea39860d05aa9275f58: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:40:55.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:55.480834 643 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:40:55.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.475366 429 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 6d464f9db55d4ea39860d05aa9275f58 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6c40c560f2cc4dbca2d2f3ca0e169156 in term 1.
01:40:55.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:55.484550 642 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 6c40c560f2cc4dbca2d2f3ca0e169156: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:40:55.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:55.484889 642 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:40:55.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.486220 558 leader_election.cc:302] T 00000000000000000000000000000000 P 6c40c560f2cc4dbca2d2f3ca0e169156 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6c40c560f2cc4dbca2d2f3ca0e169156, 6d464f9db55d4ea39860d05aa9275f58; no voters:
01:40:55.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.493275 621 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 6c40c560f2cc4dbca2d2f3ca0e169156 [term 1 FOLLOWER]: Leader election won for term 1
01:40:55.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.514432 621 raft_consensus.cc:686] T 00000000000000000000000000000000 P 6c40c560f2cc4dbca2d2f3ca0e169156 [term 1 LEADER]: Becoming Leader. State: Replica: 6c40c560f2cc4dbca2d2f3ca0e169156, State: Running, Role: LEADER
01:40:55.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.413259 531 master_runner.cc:186] Error getting master registration for 127.0.93.189:33795: OK, instance_id { permanent_uuid: "060ada4773214290b7ff6c513dd42334" instance_seqno: 1790646053408657 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
01:40:55.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.516916 531 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:40:55.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.398595 386 master_runner.cc:186] Error getting master registration for 127.0.93.189:33795: OK, instance_id { permanent_uuid: "060ada4773214290b7ff6c513dd42334" instance_seqno: 1790646053408657 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
01:40:55.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.518304 386 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:40:55.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.515797 621 consensus_queue.cc:237] T 00000000000000000000000000000000 P 6c40c560f2cc4dbca2d2f3ca0e169156 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d464f9db55d4ea39860d05aa9275f58" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 34177 } } peers { permanent_uuid: "060ada4773214290b7ff6c513dd42334" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33795 } } peers { permanent_uuid: "6c40c560f2cc4dbca2d2f3ca0e169156" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40257 } }
01:40:55.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.540132 646 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6c40c560f2cc4dbca2d2f3ca0e169156 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6c40c560f2cc4dbca2d2f3ca0e169156. Latest consensus state: current_term: 1 leader_uuid: "6c40c560f2cc4dbca2d2f3ca0e169156" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d464f9db55d4ea39860d05aa9275f58" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 34177 } } peers { permanent_uuid: "060ada4773214290b7ff6c513dd42334" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33795 } } peers { permanent_uuid: "6c40c560f2cc4dbca2d2f3ca0e169156" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40257 } } }
01:40:55.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.540997 646 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6c40c560f2cc4dbca2d2f3ca0e169156 [sys.catalog]: This master's current role is: LEADER
01:40:55.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.544060 648 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
01:40:55.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.552316 648 catalog_manager.cc:1484] Initializing Kudu cluster ID...
01:40:55.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.584333 429 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 6d464f9db55d4ea39860d05aa9275f58 [term 1 FOLLOWER]: Refusing update from remote peer 6c40c560f2cc4dbca2d2f3ca0e169156: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
01:40:55.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.591423 646 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 6c40c560f2cc4dbca2d2f3ca0e169156 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d464f9db55d4ea39860d05aa9275f58" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 34177 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.002s
01:40:55.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.625617 620 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6d464f9db55d4ea39860d05aa9275f58 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6c40c560f2cc4dbca2d2f3ca0e169156. Latest consensus state: current_term: 1 leader_uuid: "6c40c560f2cc4dbca2d2f3ca0e169156" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d464f9db55d4ea39860d05aa9275f58" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 34177 } } peers { permanent_uuid: "060ada4773214290b7ff6c513dd42334" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33795 } } peers { permanent_uuid: "6c40c560f2cc4dbca2d2f3ca0e169156" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40257 } } }
01:40:55.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.627302 620 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6d464f9db55d4ea39860d05aa9275f58 [sys.catalog]: This master's current role is: FOLLOWER
01:40:55.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.638446 649 mvcc.cc:205] Tried to move back new op lower bound from 7334486243654074368 to 7334486243445084160. Current Snapshot: MvccSnapshot[applied={T|T < 7334486243654074368}]
01:40:55.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.640451 621 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6c40c560f2cc4dbca2d2f3ca0e169156 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "6c40c560f2cc4dbca2d2f3ca0e169156" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d464f9db55d4ea39860d05aa9275f58" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 34177 } } peers { permanent_uuid: "060ada4773214290b7ff6c513dd42334" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33795 } } peers { permanent_uuid: "6c40c560f2cc4dbca2d2f3ca0e169156" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40257 } } }
01:40:55.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.641211 621 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6c40c560f2cc4dbca2d2f3ca0e169156 [sys.catalog]: This master's current role is: LEADER
01:40:55.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.643177 648 catalog_manager.cc:1347] Generated new cluster ID: f62105df3b4f48ef955bb78c531646ea
01:40:55.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.643455 648 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
01:40:55.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.654152 620 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6d464f9db55d4ea39860d05aa9275f58 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "6c40c560f2cc4dbca2d2f3ca0e169156" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d464f9db55d4ea39860d05aa9275f58" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 34177 } } peers { permanent_uuid: "060ada4773214290b7ff6c513dd42334" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33795 } } peers { permanent_uuid: "6c40c560f2cc4dbca2d2f3ca0e169156" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40257 } } }
01:40:55.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.656370 620 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6d464f9db55d4ea39860d05aa9275f58 [sys.catalog]: This master's current role is: FOLLOWER
01:40:55.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.731204 648 catalog_manager.cc:1370] Generated new certificate authority record
01:40:55.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.740645 648 catalog_manager.cc:1504] Loading token signing keys...
01:40:55.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.782531 648 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 6c40c560f2cc4dbca2d2f3ca0e169156: Generated new TSK 0
01:40:55.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.783806 648 catalog_manager.cc:1514] Initializing in-progress tserver states...
01:40:55.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.786743 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40257 } attempt: 7
01:40:55.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.809685 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-task5IFP5y/build/tsan/bin/kudu
01:40:55.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5IFP5y/build/tsan/bin/kudu
01:40:55.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/ts-0/wal
01:40:55.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/ts-0/data
01:40:55.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:40:55.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:40:55.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:40:55.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:40:55.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:40:55.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:40:55.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/ts-0/logs
01:40:55.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/ts-0/data/info.pb
01:40:55.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:40:55.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:40:55.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:40:55.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:40:55.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:40:55.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:40:55.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:40:55.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:40:55.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
01:40:55.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:40:55.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.129:0
01:40:55.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.129
01:40:55.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.129
01:40:55.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:40:55.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:34177,127.0.93.189:33795,127.0.93.188:40257
01:40:55.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:37611
01:40:55.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:40:55.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:40:55.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
01:40:55.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.844448 529 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 060ada4773214290b7ff6c513dd42334: Bootstrap starting.
01:40:55.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.852820 529 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 060ada4773214290b7ff6c513dd42334: Neither blocks nor log segments found. Creating new log.
01:40:55.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.855495 529 log.cc:824] T 00000000000000000000000000000000 P 060ada4773214290b7ff6c513dd42334: Log is configured to *not* fsync() on all Append() calls
01:40:55.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.879544 529 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 060ada4773214290b7ff6c513dd42334: No bootstrap required, opened a new log
01:40:55.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.960542 529 raft_consensus.cc:348] T 00000000000000000000000000000000 P 060ada4773214290b7ff6c513dd42334 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d464f9db55d4ea39860d05aa9275f58" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 34177 } } peers { permanent_uuid: "060ada4773214290b7ff6c513dd42334" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33795 } } peers { permanent_uuid: "6c40c560f2cc4dbca2d2f3ca0e169156" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40257 } }
01:40:55.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.963018 529 raft_consensus.cc:374] T 00000000000000000000000000000000 P 060ada4773214290b7ff6c513dd42334 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
01:40:55.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.963984 529 raft_consensus.cc:729] T 00000000000000000000000000000000 P 060ada4773214290b7ff6c513dd42334 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 060ada4773214290b7ff6c513dd42334, State: Initialized, Role: FOLLOWER
01:40:55.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.966624 529 consensus_queue.cc:260] T 00000000000000000000000000000000 P 060ada4773214290b7ff6c513dd42334 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d464f9db55d4ea39860d05aa9275f58" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 34177 } } peers { permanent_uuid: "060ada4773214290b7ff6c513dd42334" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33795 } } peers { permanent_uuid: "6c40c560f2cc4dbca2d2f3ca0e169156" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40257 } }
01:40:55.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.984938 657 sys_catalog.cc:455] T 00000000000000000000000000000000 P 060ada4773214290b7ff6c513dd42334 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d464f9db55d4ea39860d05aa9275f58" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 34177 } } peers { permanent_uuid: "060ada4773214290b7ff6c513dd42334" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33795 } } peers { permanent_uuid: "6c40c560f2cc4dbca2d2f3ca0e169156" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40257 } } }
01:40:55.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.985818 657 sys_catalog.cc:458] T 00000000000000000000000000000000 P 060ada4773214290b7ff6c513dd42334 [sys.catalog]: This master's current role is: FOLLOWER
01:40:55.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.991271 504 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6c40c560f2cc4dbca2d2f3ca0e169156" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "060ada4773214290b7ff6c513dd42334" is_pre_election: true
01:40:55.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.995276 503 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6c40c560f2cc4dbca2d2f3ca0e169156" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "060ada4773214290b7ff6c513dd42334"
01:40:56.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.992179 504 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 060ada4773214290b7ff6c513dd42334 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6c40c560f2cc4dbca2d2f3ca0e169156 in term 0.
01:40:56.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:55.999218 502 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 060ada4773214290b7ff6c513dd42334 [term 0 FOLLOWER]: Advancing to term 1
01:40:56.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:56.010002 502 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 060ada4773214290b7ff6c513dd42334 [term 1 FOLLOWER]: Refusing update from remote peer 6c40c560f2cc4dbca2d2f3ca0e169156: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
01:40:56.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:56.016191 529 sys_catalog.cc:564] T 00000000000000000000000000000000 P 060ada4773214290b7ff6c513dd42334 [sys.catalog]: configured and running, proceeding with master startup.
01:40:56.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:56.016559 646 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 6c40c560f2cc4dbca2d2f3ca0e169156 [LEADER]: Connected to new peer: Peer: permanent_uuid: "060ada4773214290b7ff6c513dd42334" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
01:40:56.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:56.089289 657 sys_catalog.cc:455] T 00000000000000000000000000000000 P 060ada4773214290b7ff6c513dd42334 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6c40c560f2cc4dbca2d2f3ca0e169156. Latest consensus state: current_term: 1 leader_uuid: "6c40c560f2cc4dbca2d2f3ca0e169156" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d464f9db55d4ea39860d05aa9275f58" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 34177 } } peers { permanent_uuid: "060ada4773214290b7ff6c513dd42334" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33795 } } peers { permanent_uuid: "6c40c560f2cc4dbca2d2f3ca0e169156" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40257 } } }
01:40:56.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:56.090046 657 sys_catalog.cc:458] T 00000000000000000000000000000000 P 060ada4773214290b7ff6c513dd42334 [sys.catalog]: This master's current role is: FOLLOWER
01:40:56.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:56.089519 663 sys_catalog.cc:455] T 00000000000000000000000000000000 P 060ada4773214290b7ff6c513dd42334 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "6c40c560f2cc4dbca2d2f3ca0e169156" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d464f9db55d4ea39860d05aa9275f58" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 34177 } } peers { permanent_uuid: "060ada4773214290b7ff6c513dd42334" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33795 } } peers { permanent_uuid: "6c40c560f2cc4dbca2d2f3ca0e169156" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40257 } } }
01:40:56.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:56.090387 663 sys_catalog.cc:458] T 00000000000000000000000000000000 P 060ada4773214290b7ff6c513dd42334 [sys.catalog]: This master's current role is: FOLLOWER
01:40:56.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:56.117883 673 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
01:40:56.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:56.355269 655 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:40:56.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:56.355810 655 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:40:56.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:56.356319 655 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:40:56.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:56.391656 655 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:40:56.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:56.392547 655 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.93.129
01:40:56.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:56.429682 655 tablet_server_runner.cc:78] Tablet server non-default flags:
01:40:56.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:40:56.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:37611
01:40:56.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:40:56.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:40:56.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/ts-0/data
01:40:56.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/ts-0/wal
01:40:56.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:40:56.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:40:56.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.129:0
01:40:56.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:40:56.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:40:56.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:40:56.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/ts-0/data/info.pb
01:40:56.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.129
01:40:56.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:40:56.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:34177,127.0.93.189:33795,127.0.93.188:40257
01:40:56.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:40:56.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:40:56.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:40:56.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:40:56.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:40:56.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.129
01:40:56.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/ts-0/logs
01:40:56.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:40:56.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:40:56.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:40:56.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
01:40:56.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:40:56.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:40:56.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:40:56.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 00:23:04 UTC on ccycloud.quasar-noncbr.root.comops.site
01:40:56.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4678933
01:40:56.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:40:56.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:56.431320 655 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:40:56.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:56.433429 655 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:40:56.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:56.483865 646 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6c40c560f2cc4dbca2d2f3ca0e169156 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "6c40c560f2cc4dbca2d2f3ca0e169156" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d464f9db55d4ea39860d05aa9275f58" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 34177 } } peers { permanent_uuid: "060ada4773214290b7ff6c513dd42334" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33795 } } peers { permanent_uuid: "6c40c560f2cc4dbca2d2f3ca0e169156" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40257 } } }
01:40:56.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:56.487133 646 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6c40c560f2cc4dbca2d2f3ca0e169156 [sys.catalog]: This master's current role is: LEADER
01:40:56.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:56.487210 643 catalog_manager.cc:1259] Loaded cluster ID: f62105df3b4f48ef955bb78c531646ea
01:40:56.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:56.487488 643 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 6d464f9db55d4ea39860d05aa9275f58: loading cluster ID for follower catalog manager: success
01:40:56.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:56.499663 681 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:40:56.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:56.504240 643 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 6d464f9db55d4ea39860d05aa9275f58: acquiring CA information for follower catalog manager: success
01:40:56.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:56.523458 643 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 6d464f9db55d4ea39860d05aa9275f58: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
01:40:57.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:57.153270 673 catalog_manager.cc:1259] Loaded cluster ID: f62105df3b4f48ef955bb78c531646ea
01:40:57.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:57.153626 673 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 060ada4773214290b7ff6c513dd42334: loading cluster ID for follower catalog manager: success
01:40:57.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:57.174538 673 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 060ada4773214290b7ff6c513dd42334: acquiring CA information for follower catalog manager: success
01:40:57.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:57.188990 673 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 060ada4773214290b7ff6c513dd42334: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
01:40:58.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:56.501626 679 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:40:58.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:56.500984 684 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:40:58.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:58.605566 682 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2103 milliseconds
01:40:58.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:58.605768 655 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
01:40:58.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:58.608839 655 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:40:58.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:58.612835 655 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:40:58.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:58.614421 655 hybrid_clock.cc:648] HybridClock initialized: now 1790646058614388 us; error 52 us; skew 500 ppm
01:40:58.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:58.615268 655 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:40:58.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:58.626992 655 webserver.cc:460] Webserver started at http://127.0.93.129:34541/ using document root <none> and password file <none>
01:40:58.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:58.628123 655 fs_manager.cc:359] Metadata directory not provided
01:40:58.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:58.628414 655 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:40:58.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:58.628944 655 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:40:58.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:58.635082 655 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/ts-0/data/instance:
01:40:58.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "078dace997254ccda75650a09432b1d5"
01:40:58.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:40:58 on dist-test-slave-rw6r"
01:40:58.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:58.636402 655 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/ts-0/wal/instance:
01:40:58.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "078dace997254ccda75650a09432b1d5"
01:40:58.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:40:58 on dist-test-slave-rw6r"
01:40:58.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:58.645640 655 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.008s sys 0.000s
01:40:58.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:58.653178 693 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:40:58.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:58.654452 655 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.001s sys 0.004s
01:40:58.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:58.654845 655 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/ts-0/data,/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/ts-0/wal
01:40:58.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "078dace997254ccda75650a09432b1d5"
01:40:58.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:40:58 on dist-test-slave-rw6r"
01:40:58.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:58.655241 655 fs_report.cc:383] FS layout report
01:40:58.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:40:58.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/ts-0/wal
01:40:58.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/ts-0/wal
01:40:58.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/ts-0/data/data
01:40:58.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:40:58.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:40:58.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:40:58.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:40:58.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:40:58.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:40:58.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:40:58.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:40:58.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:40:58.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:40:58.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:40:58.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:58.742108 655 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:40:58.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:58.744092 655 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:40:58.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:58.744738 655 kserver.cc:163] Server-wide thread pool size limit: 3276
01:40:58.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:58.748148 655 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
01:40:58.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:58.754737 655 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
01:40:58.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:58.755029 655 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
01:40:58.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:58.755326 655 ts_tablet_manager.cc:610] Registered 0 tablets
01:40:58.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:58.755527 655 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
01:40:58.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:58.983893 655 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.129:39535
01:40:58.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:58.983996 805 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.129:39535 every 8 connection(s)
01:40:58.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:58.986845 655 server_base.cc:1166] Dumped server information to /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/ts-0/data/info.pb
01:40:58.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:58.997079 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-task5IFP5y/build/tsan/bin/kudu as pid 655
01:40:59.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:58.997661 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/ts-0/wal/instance
01:40:59.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:59.024350 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-task5IFP5y/build/tsan/bin/kudu
01:40:59.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5IFP5y/build/tsan/bin/kudu
01:40:59.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/ts-1/wal
01:40:59.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/ts-1/data
01:40:59.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:40:59.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:40:59.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:40:59.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:40:59.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:40:59.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:40:59.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/ts-1/logs
01:40:59.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/ts-1/data/info.pb
01:40:59.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:40:59.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:40:59.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:40:59.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:40:59.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:40:59.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:40:59.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:40:59.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:40:59.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
01:40:59.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:40:59.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.130:0
01:40:59.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.130
01:40:59.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.130
01:40:59.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:40:59.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:34177,127.0.93.189:33795,127.0.93.188:40257
01:40:59.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:37611
01:40:59.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:40:59.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:40:59.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
01:40:59.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:59.120157 806 heartbeater.cc:347] Connected to a master server at 127.0.93.188:40257
01:40:59.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:59.120647 806 heartbeater.cc:464] Registering TS with master...
01:40:59.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:59.121714 806 heartbeater.cc:511] Master 127.0.93.188:40257 requested a full tablet report, sending...
01:40:59.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:59.125115 573 ts_manager.cc:194] Registered new tserver with Master: 078dace997254ccda75650a09432b1d5 (127.0.93.129:39535)
01:40:59.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:59.128326 573 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.93.129:43903
01:40:59.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:59.135324 810 heartbeater.cc:347] Connected to a master server at 127.0.93.190:34177
01:40:59.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:59.135977 810 heartbeater.cc:464] Registering TS with master...
01:40:59.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:59.136765 810 heartbeater.cc:511] Master 127.0.93.190:34177 requested a full tablet report, sending...
01:40:59.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:59.140017 419 ts_manager.cc:194] Registered new tserver with Master: 078dace997254ccda75650a09432b1d5 (127.0.93.129:39535)
01:40:59.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:59.140594 807 heartbeater.cc:347] Connected to a master server at 127.0.93.189:33795
01:40:59.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:59.140929 807 heartbeater.cc:464] Registering TS with master...
01:40:59.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:59.141680 807 heartbeater.cc:511] Master 127.0.93.189:33795 requested a full tablet report, sending...
01:40:59.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:59.145021 493 ts_manager.cc:194] Registered new tserver with Master: 078dace997254ccda75650a09432b1d5 (127.0.93.129:39535)
01:40:59.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:59.475800 813 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:40:59.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:59.476346 813 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:40:59.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:59.476969 813 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:40:59.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:59.509338 813 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:40:59.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:59.510341 813 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.93.130
01:40:59.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:59.546286 813 tablet_server_runner.cc:78] Tablet server non-default flags:
01:40:59.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:40:59.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:37611
01:40:59.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:40:59.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:40:59.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/ts-1/data
01:40:59.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/ts-1/wal
01:40:59.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:40:59.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:40:59.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.130:0
01:40:59.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:40:59.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:40:59.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:40:59.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/ts-1/data/info.pb
01:40:59.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.130
01:40:59.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:40:59.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:34177,127.0.93.189:33795,127.0.93.188:40257
01:40:59.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:40:59.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:40:59.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:40:59.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:40:59.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:40:59.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.130
01:40:59.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/ts-1/logs
01:40:59.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:40:59.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:40:59.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:40:59.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
01:40:59.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:40:59.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:40:59.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:40:59.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 00:23:04 UTC on ccycloud.quasar-noncbr.root.comops.site
01:40:59.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4678933
01:40:59.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:40:59.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:59.547950 813 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:40:59.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:59.550169 813 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:40:59.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:59.574633 822 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:41:00.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:00.133363 806 heartbeater.cc:503] Master 127.0.93.188:40257 was elected leader, sending a full tablet report...
01:41:00.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:59.574641 823 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:41:00.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:40:59.575505 825 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:41:00.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:40:59.579272 813 server_base.cc:1034] running on GCE node
01:41:00.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:00.803539 813 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:41:00.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:00.805888 813 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:41:00.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:00.807252 813 hybrid_clock.cc:648] HybridClock initialized: now 1790646060807203 us; error 61 us; skew 500 ppm
01:41:00.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:00.808200 813 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:41:00.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:00.814134 813 webserver.cc:460] Webserver started at http://127.0.93.130:32801/ using document root <none> and password file <none>
01:41:00.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:00.815023 813 fs_manager.cc:359] Metadata directory not provided
01:41:00.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:00.815222 813 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:41:00.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:00.815593 813 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:41:00.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:00.820778 813 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/ts-1/data/instance:
01:41:00.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f413a0988bab4039a6feab6d6a9d3eda"
01:41:00.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:41:00 on dist-test-slave-rw6r"
01:41:00.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:00.821993 813 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/ts-1/wal/instance:
01:41:00.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f413a0988bab4039a6feab6d6a9d3eda"
01:41:00.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:41:00 on dist-test-slave-rw6r"
01:41:00.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:00.828926 813 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.007s sys 0.001s
01:41:00.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:00.834225 834 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:41:00.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:00.835279 813 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
01:41:00.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:00.835556 813 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/ts-1/data,/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/ts-1/wal
01:41:00.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f413a0988bab4039a6feab6d6a9d3eda"
01:41:00.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:41:00 on dist-test-slave-rw6r"
01:41:00.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:00.835876 813 fs_report.cc:383] FS layout report
01:41:00.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:41:00.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/ts-1/wal
01:41:00.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/ts-1/wal
01:41:00.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/ts-1/data/data
01:41:00.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:41:00.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:41:00.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:41:00.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:41:00.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:41:00.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:41:00.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:41:00.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:41:00.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:41:00.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:41:00.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:41:00.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:00.875761 813 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:41:00.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:00.877239 813 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:41:00.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:00.877637 813 kserver.cc:163] Server-wide thread pool size limit: 3276
01:41:00.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:00.880082 813 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
01:41:00.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:00.884706 813 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
01:41:00.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:00.884940 813 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
01:41:00.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:00.885234 813 ts_tablet_manager.cc:610] Registered 0 tablets
01:41:00.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:00.885424 813 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
01:41:01.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:01.039259 813 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.130:34583
01:41:01.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:01.039397 946 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.130:34583 every 8 connection(s)
01:41:01.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:01.041904 813 server_base.cc:1166] Dumped server information to /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/ts-1/data/info.pb
01:41:01.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:01.051216 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-task5IFP5y/build/tsan/bin/kudu as pid 813
01:41:01.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:01.052100 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/ts-1/wal/instance
01:41:01.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:01.073122 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-task5IFP5y/build/tsan/bin/kudu
01:41:01.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5IFP5y/build/tsan/bin/kudu
01:41:01.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/ts-2/wal
01:41:01.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/ts-2/data
01:41:01.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:41:01.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:41:01.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:41:01.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:41:01.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:41:01.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:41:01.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/ts-2/logs
01:41:01.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/ts-2/data/info.pb
01:41:01.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:41:01.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:41:01.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:41:01.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:41:01.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:41:01.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:41:01.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:41:01.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:41:01.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
01:41:01.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:41:01.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.131:0
01:41:01.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.131
01:41:01.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.131
01:41:01.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:41:01.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:34177,127.0.93.189:33795,127.0.93.188:40257
01:41:01.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:37611
01:41:01.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:41:01.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:41:01.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
01:41:01.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:01.101127 948 heartbeater.cc:347] Connected to a master server at 127.0.93.189:33795
01:41:01.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:01.101629 948 heartbeater.cc:464] Registering TS with master...
01:41:01.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:01.102926 948 heartbeater.cc:511] Master 127.0.93.189:33795 requested a full tablet report, sending...
01:41:01.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:01.103727 947 heartbeater.cc:347] Connected to a master server at 127.0.93.188:40257
01:41:01.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:01.104034 947 heartbeater.cc:464] Registering TS with master...
01:41:01.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:01.104744 949 heartbeater.cc:347] Connected to a master server at 127.0.93.190:34177
01:41:01.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:01.104770 947 heartbeater.cc:511] Master 127.0.93.188:40257 requested a full tablet report, sending...
01:41:01.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:01.105068 949 heartbeater.cc:464] Registering TS with master...
01:41:01.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:01.105788 949 heartbeater.cc:511] Master 127.0.93.190:34177 requested a full tablet report, sending...
01:41:01.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:01.106966 494 ts_manager.cc:194] Registered new tserver with Master: f413a0988bab4039a6feab6d6a9d3eda (127.0.93.130:34583)
01:41:01.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:01.107157 573 ts_manager.cc:194] Registered new tserver with Master: f413a0988bab4039a6feab6d6a9d3eda (127.0.93.130:34583)
01:41:01.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:01.109007 573 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.93.130:42807
01:41:01.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:01.109485 419 ts_manager.cc:194] Registered new tserver with Master: f413a0988bab4039a6feab6d6a9d3eda (127.0.93.130:34583)
01:41:01.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:01.397408 957 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:41:01.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:01.397989 957 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:41:01.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:01.398562 957 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:41:01.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:01.428717 957 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:41:01.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:01.429682 957 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.93.131
01:41:01.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:01.464844 957 tablet_server_runner.cc:78] Tablet server non-default flags:
01:41:01.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:41:01.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:37611
01:41:01.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:41:01.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:41:01.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/ts-2/data
01:41:01.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/ts-2/wal
01:41:01.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:41:01.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:41:01.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.131:0
01:41:01.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:41:01.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:41:01.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:41:01.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/ts-2/data/info.pb
01:41:01.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.131
01:41:01.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:41:01.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:34177,127.0.93.189:33795,127.0.93.188:40257
01:41:01.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:41:01.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:41:01.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:41:01.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:41:01.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:41:01.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.131
01:41:01.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/ts-2/logs
01:41:01.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:41:01.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:41:01.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:41:01.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
01:41:01.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:41:01.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:41:01.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:41:01.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 00:23:04 UTC on ccycloud.quasar-noncbr.root.comops.site
01:41:01.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4678933
01:41:01.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:41:01.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:01.466457 957 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:41:01.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:01.468168 957 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:41:01.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:01.488845 963 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:41:02.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:02.113057 947 heartbeater.cc:503] Master 127.0.93.188:40257 was elected leader, sending a full tablet report...
01:41:02.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:01.489434 964 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:41:02.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:01.493629 957 server_base.cc:1034] running on GCE node
01:41:02.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:01.492147 966 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:41:02.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:02.689194 957 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:41:02.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:02.691603 957 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:41:02.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:02.693051 957 hybrid_clock.cc:648] HybridClock initialized: now 1790646062693016 us; error 50 us; skew 500 ppm
01:41:02.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:02.693868 957 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:41:02.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:02.701046 957 webserver.cc:460] Webserver started at http://127.0.93.131:33543/ using document root <none> and password file <none>
01:41:02.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:02.701949 957 fs_manager.cc:359] Metadata directory not provided
01:41:02.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:02.702176 957 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:41:02.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:02.702646 957 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:41:02.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:02.707468 957 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/ts-2/data/instance:
01:41:02.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d96379350f944a9ca6b782758efeef9c"
01:41:02.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:41:02 on dist-test-slave-rw6r"
01:41:02.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:02.708621 957 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/ts-2/wal/instance:
01:41:02.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d96379350f944a9ca6b782758efeef9c"
01:41:02.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:41:02 on dist-test-slave-rw6r"
01:41:02.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:02.715164 957 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.006s sys 0.000s
01:41:02.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:02.720556 974 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:41:02.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:02.721575 957 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
01:41:02.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:02.721908 957 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/ts-2/data,/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/ts-2/wal
01:41:02.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d96379350f944a9ca6b782758efeef9c"
01:41:02.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:41:02 on dist-test-slave-rw6r"
01:41:02.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:02.722272 957 fs_report.cc:383] FS layout report
01:41:02.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:41:02.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/ts-2/wal
01:41:02.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/ts-2/wal
01:41:02.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/ts-2/data/data
01:41:02.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:41:02.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:41:02.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:41:02.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:41:02.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:41:02.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:41:02.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:41:02.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:41:02.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:41:02.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:41:02.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:41:02.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:02.760360 957 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:41:02.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:02.761895 957 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:41:02.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:02.762419 957 kserver.cc:163] Server-wide thread pool size limit: 3276
01:41:02.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:02.765362 957 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
01:41:02.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:02.769466 957 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
01:41:02.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:02.769709 957 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
01:41:02.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:02.770053 957 ts_tablet_manager.cc:610] Registered 0 tablets
01:41:02.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:02.770246 957 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
01:41:02.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:02.923308 957 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.131:33511
01:41:02.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:02.923420 1086 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.131:33511 every 8 connection(s)
01:41:02.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:02.926102 957 server_base.cc:1166] Dumped server information to /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/ts-2/data/info.pb
01:41:02.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:02.935429 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-task5IFP5y/build/tsan/bin/kudu as pid 957
01:41:02.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:02.936098 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster2805780575179909391/ts-2/wal/instance
01:41:03.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:03.000378 1087 heartbeater.cc:347] Connected to a master server at 127.0.93.188:40257
01:41:03.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:03.000774 1087 heartbeater.cc:464] Registering TS with master...
01:41:03.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:03.002727 1087 heartbeater.cc:511] Master 127.0.93.188:40257 requested a full tablet report, sending...
01:41:03.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:03.004283 1088 heartbeater.cc:347] Connected to a master server at 127.0.93.189:33795
01:41:03.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:03.004627 1088 heartbeater.cc:464] Registering TS with master...
01:41:03.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:03.005862 1088 heartbeater.cc:511] Master 127.0.93.189:33795 requested a full tablet report, sending...
01:41:03.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:03.006319 573 ts_manager.cc:194] Registered new tserver with Master: d96379350f944a9ca6b782758efeef9c (127.0.93.131:33511)
01:41:03.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:03.008270 573 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.93.131:40475
01:41:03.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:03.008366 494 ts_manager.cc:194] Registered new tserver with Master: d96379350f944a9ca6b782758efeef9c (127.0.93.131:33511)
01:41:03.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:03.011466 1089 heartbeater.cc:347] Connected to a master server at 127.0.93.190:34177
01:41:03.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:03.011832 1089 heartbeater.cc:464] Registering TS with master...
01:41:03.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:03.012502 1089 heartbeater.cc:511] Master 127.0.93.190:34177 requested a full tablet report, sending...
01:41:03.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:03.014736 419 ts_manager.cc:194] Registered new tserver with Master: d96379350f944a9ca6b782758efeef9c (127.0.93.131:33511)
01:41:03.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:03.016803 374 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
01:41:03.027 [DEBUG - main] (MiniKuduCluster.java:195) Response:
01:41:03.029 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
01:41:03.039 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.0.93.190"
port: 34177
}
bound_http_address {
host: "127.0.93.190"
port: 46333
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.0.93.189"
port: 33795
}
bound_http_address {
host: "127.0.93.189"
port: 43287
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.0.93.188"
port: 40257
}
bound_http_address {
host: "127.0.93.188"
port: 38599
}
}
}
01:41:03.046 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
01:41:03.049 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.0.93.129"
port: 39535
}
bound_http_address {
host: "127.0.93.129"
port: 34541
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.0.93.130"
port: 34583
}
bound_http_address {
host: "127.0.93.130"
port: 32801
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.0.93.131"
port: 33511
}
bound_http_address {
host: "127.0.93.131"
port: 33543
}
}
}
01:41:03.050 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
01:41:03.235 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
01:41:03.360 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task5IFP5y/test-tmp
01:41:03.361 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
01:41:03.361 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task5IFP5y/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
01:41:03.364 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
01:41:03.714 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:41:03.715 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
01:41:03.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:03.718160 1101 mini_oidc.cc:152] Starting JWKS server
01:41:03.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:03.722671 1101 webserver.cc:460] Webserver started at http://127.0.0.1:45993/ using document root <none> and password file <none>
01:41:03.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:03.723116 1101 mini_oidc.cc:189] Starting OIDC Discovery server
01:41:03.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:03.725378 1101 webserver.cc:460] Webserver started at http://127.0.0.1:36929/ using document root <none> and password file <none>
01:41:03.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
01:41:03.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/krb5kdc/principal' for realm 'KRBTEST.COM',
01:41:03.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
01:41:03.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 01:41:03 dist-test-slave-rw6r krb5kdc[1114](info): setting up network...
01:41:03.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
01:41:03.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 01:41:03 dist-test-slave-rw6r krb5kdc[1114](info): set up 2 sockets
01:41:03.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 01:41:03 dist-test-slave-rw6r krb5kdc[1114](info): commencing operation
01:41:03.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
01:41:04.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:04.012985 1087 heartbeater.cc:503] Master 127.0.93.188:40257 was elected leader, sending a full tablet report...
01:41:04.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:04.843281 1101 mini_kdc.cc:121] Time spent starting KDC: real 1.089s user 0.003s sys 0.016s
01:41:04.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
01:41:04.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
01:41:04.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
01:41:04.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
01:41:04.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
01:41:04.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
01:41:04.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
01:41:04.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
01:41:04.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
01:41:04.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
01:41:04.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/krb5kdc/test-user.keytab.
01:41:04.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/krb5kdc/test-user.keytab.
01:41:04.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 01:41:04 dist-test-slave-rw6r krb5kdc[1114](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790646064, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:41:04.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
01:41:04.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T01:41:04Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
01:41:04.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T01:41:04Z Disabled control of system clock
01:41:05.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.19.126@KRBTEST.COM; defaulting to no policy
01:41:05.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:41:05.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.19.126@KRBTEST.COM" created.
01:41:05.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:41:05.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/krb5kdc/kudu_127.1.19.126.keytab.
01:41:05.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/krb5kdc/kudu_127.1.19.126.keytab.
01:41:05.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:05.040843 1101 external_mini_cluster.cc:1326] Running /tmp/dist-test-task5IFP5y/build/tsan/bin/kudu
01:41:05.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5IFP5y/build/tsan/bin/kudu
01:41:05.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-0/wal
01:41:05.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-0/data
01:41:05.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:41:05.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:41:05.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:41:05.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:41:05.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:41:05.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:41:05.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-0/logs
01:41:05.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-0/data/info.pb
01:41:05.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:41:05.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:41:05.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:41:05.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:41:05.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:41:05.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:41:05.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:41:05.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:41:05.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:41:05.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:41:05.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:41:05.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:41:05.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.126:40005
01:41:05.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.126
01:41:05.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:41:05.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/krb5kdc/kudu_127.1.19.126.keytab
01:41:05.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.126
01:41:05.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:41:05.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:41:05.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:41:05.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.84:38321
01:41:05.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:41:05.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:41:05.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:41:05.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:41:05.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.126:40005,127.1.19.125:39053,127.1.19.124:44787
01:41:05.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:41:05.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45993/jwks.json
01:41:05.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:41:05.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:05.349149 1127 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:41:05.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:05.349701 1127 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:41:05.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:05.350350 1127 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:41:05.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:05.382392 1127 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:41:05.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:05.382645 1127 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:41:05.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:05.382896 1127 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:41:05.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:05.383108 1127 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:41:05.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:05.383293 1127 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:41:05.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:05.383476 1127 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:41:05.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:05.383611 1127 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45993/jwks.json
01:41:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:05.420846 1127 master_runner.cc:386] Master server non-default flags:
01:41:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:41:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.84:38321
01:41:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:41:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:41:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-0/data
01:41:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-0/wal
01:41:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:41:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:41:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.126:40005,127.1.19.125:39053,127.1.19.124:44787
01:41:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/krb5kdc/kudu_127.1.19.126.keytab
01:41:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.126
01:41:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:41:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:41:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:41:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.126:40005
01:41:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:41:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:41:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:41:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45993/jwks.json
01:41:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:41:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:41:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:41:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:41:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:41:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-0/data/info.pb
01:41:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.126
01:41:05.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:41:05.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:41:05.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:41:05.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:41:05.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:41:05.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:41:05.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-0/logs
01:41:05.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:41:05.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:41:05.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:41:05.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:41:05.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:41:05.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:41:05.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:41:05.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 00:23:04 UTC on ccycloud.quasar-noncbr.root.comops.site
01:41:05.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4678933
01:41:05.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:41:05.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:05.421998 1127 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:41:05.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:05.423235 1127 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:41:05.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:05.447543 1133 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:41:06.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:05.450459 1136 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:41:06.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:05.447713 1134 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:41:06.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:06.674347 1135 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
01:41:06.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:06.674356 1127 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
01:41:06.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:06.678310 1127 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:41:06.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:06.681629 1127 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:41:06.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:06.683060 1127 hybrid_clock.cc:648] HybridClock initialized: now 1790646066682990 us; error 80 us; skew 500 ppm
01:41:06.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:06.683928 1127 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:41:06.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 01:41:06 dist-test-slave-rw6r krb5kdc[1114](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790646066, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:41:06.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:06.689980 1127 init.cc:377] Logged in from keytab as kudu/127.1.19.126@KRBTEST.COM (short username kudu)
01:41:06.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:06.695997 1127 webserver.cc:460] Webserver started at http://127.1.19.126:42113/ using document root <none> and password file <none>
01:41:06.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:06.696826 1127 fs_manager.cc:359] Metadata directory not provided
01:41:06.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:06.697016 1127 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:41:06.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:06.697391 1127 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:41:06.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:06.702006 1127 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-0/data/instance:
01:41:06.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "85a011fbef4b438c84f15e8e42b174c8"
01:41:06.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:41:06 on dist-test-slave-rw6r"
01:41:06.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:06.703298 1127 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-0/wal/instance:
01:41:06.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "85a011fbef4b438c84f15e8e42b174c8"
01:41:06.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:41:06 on dist-test-slave-rw6r"
01:41:06.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:06.711000 1127 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.002s sys 0.005s
01:41:06.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:06.716670 1146 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:41:06.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:06.717805 1127 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
01:41:06.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:06.718075 1127 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-0/data,/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-0/wal
01:41:06.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "85a011fbef4b438c84f15e8e42b174c8"
01:41:06.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:41:06 on dist-test-slave-rw6r"
01:41:06.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:06.718355 1127 fs_report.cc:383] FS layout report
01:41:06.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:41:06.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-0/wal
01:41:06.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-0/wal
01:41:06.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-0/data/data
01:41:06.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:41:06.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:41:06.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:41:06.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:41:06.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:41:06.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:41:06.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:41:06.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:41:06.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:41:06.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:41:06.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:41:06.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:06.761740 1127 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:41:06.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:06.768071 1127 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:41:06.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:06.768481 1127 kserver.cc:163] Server-wide thread pool size limit: 3276
01:41:06.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:06.847045 1127 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.126:40005
01:41:06.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:06.847114 1197 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.126:40005 every 8 connection(s)
01:41:06.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:06.850112 1127 server_base.cc:1166] Dumped server information to /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-0/data/info.pb
01:41:06.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:06.856699 1101 external_mini_cluster.cc:1388] Started /tmp/dist-test-task5IFP5y/build/tsan/bin/kudu as pid 1127
01:41:06.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:06.857234 1101 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-0/wal/instance
01:41:06.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:06.857780 1198 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:41:06.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:06.874151 1198 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.126" port: 40005 } has no permanent_uuid. Determining permanent_uuid...
01:41:06.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.19.125@KRBTEST.COM; defaulting to no policy
01:41:06.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:41:06.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.19.125@KRBTEST.COM" created.
01:41:06.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 01:41:06 dist-test-slave-rw6r krb5kdc[1114](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790646066, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.126@KRBTEST.COM for kudu/127.1.19.126@KRBTEST.COM
01:41:06.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:41:06.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/krb5kdc/kudu_127.1.19.125.keytab.
01:41:06.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/krb5kdc/kudu_127.1.19.125.keytab.
01:41:06.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:06.905952 1101 external_mini_cluster.cc:1326] Running /tmp/dist-test-task5IFP5y/build/tsan/bin/kudu
01:41:06.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5IFP5y/build/tsan/bin/kudu
01:41:06.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-1/wal
01:41:06.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-1/data
01:41:06.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:41:06.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:41:06.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:41:06.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:41:06.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:41:06.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:41:06.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-1/logs
01:41:06.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-1/data/info.pb
01:41:06.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:41:06.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:41:06.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:41:06.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:41:06.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:41:06.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:41:06.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:41:06.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:41:06.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:41:06.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:41:06.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:41:06.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:41:06.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.125:39053
01:41:06.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.125
01:41:06.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:41:06.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/krb5kdc/kudu_127.1.19.125.keytab
01:41:06.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.125
01:41:06.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:41:06.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:41:06.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:41:06.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.84:38321
01:41:06.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:41:06.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:41:06.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:41:06.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:41:06.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.126:40005,127.1.19.125:39053,127.1.19.124:44787
01:41:06.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:41:06.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45993/jwks.json
01:41:06.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:41:06.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:06.915050 1198 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.125" port: 39053 } has no permanent_uuid. Determining permanent_uuid...
01:41:06.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:06.918025 1147 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.125:39053: connect: Connection refused (error 111)
01:41:06.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:06.922492 1198 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.125:39053: Network error: Client connection negotiation failed: client connection to 127.1.19.125:39053: connect: Connection refused (error 111)
01:41:06.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:06.972015 1198 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.125" port: 39053 } attempt: 1
01:41:06.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:06.977221 1198 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.125:39053: Network error: Client connection negotiation failed: client connection to 127.1.19.125:39053: connect: Connection refused (error 111)
01:41:07.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:07.045619 1198 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.125" port: 39053 } attempt: 2
01:41:07.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:07.049729 1198 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.125:39053: Network error: Client connection negotiation failed: client connection to 127.1.19.125:39053: connect: Connection refused (error 111)
01:41:07.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:07.141166 1198 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.125" port: 39053 } attempt: 3
01:41:07.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:07.145224 1198 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.125:39053: Network error: Client connection negotiation failed: client connection to 127.1.19.125:39053: connect: Connection refused (error 111)
01:41:07.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:07.258302 1204 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:41:07.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:07.258916 1204 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:41:07.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:07.259481 1204 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:41:07.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:07.288605 1198 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.125" port: 39053 } attempt: 4
01:41:07.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:07.291325 1204 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:41:07.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:07.291635 1204 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:41:07.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:07.291980 1204 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:41:07.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:07.292326 1204 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:41:07.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:07.292230 1198 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.125:39053: Network error: Client connection negotiation failed: client connection to 127.1.19.125:39053: connect: Connection refused (error 111)
01:41:07.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:07.292564 1204 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:41:07.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:07.292759 1204 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:41:07.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:07.292896 1204 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45993/jwks.json
01:41:07.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:07.328310 1204 master_runner.cc:386] Master server non-default flags:
01:41:07.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:41:07.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.84:38321
01:41:07.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:41:07.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:41:07.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-1/data
01:41:07.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-1/wal
01:41:07.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:41:07.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:41:07.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.126:40005,127.1.19.125:39053,127.1.19.124:44787
01:41:07.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/krb5kdc/kudu_127.1.19.125.keytab
01:41:07.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.125
01:41:07.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:41:07.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:41:07.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:41:07.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.125:39053
01:41:07.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:41:07.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:41:07.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:41:07.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45993/jwks.json
01:41:07.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:41:07.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:41:07.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:41:07.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:41:07.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:41:07.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-1/data/info.pb
01:41:07.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.125
01:41:07.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:41:07.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:41:07.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:41:07.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:41:07.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:41:07.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:41:07.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-1/logs
01:41:07.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:41:07.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:41:07.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:41:07.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:41:07.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:41:07.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:41:07.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:41:07.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 00:23:04 UTC on ccycloud.quasar-noncbr.root.comops.site
01:41:07.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4678933
01:41:07.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:41:07.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:07.329401 1204 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:41:07.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:07.330312 1204 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:41:07.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:07.349527 1214 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:41:07.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:07.591717 1198 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.125" port: 39053 } attempt: 5
01:41:07.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:07.595610 1198 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.125:39053: Network error: Client connection negotiation failed: client connection to 127.1.19.125:39053: connect: Connection refused (error 111)
01:41:08.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:08.143225 1198 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.125" port: 39053 } attempt: 6
01:41:08.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:08.150549 1198 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.125:39053: Network error: Client connection negotiation failed: client connection to 127.1.19.125:39053: connect: Connection refused (error 111)
01:41:08.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:07.349671 1212 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:41:08.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:07.350395 1211 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:41:08.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:08.615370 1213 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
01:41:08.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:08.615935 1204 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
01:41:08.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:08.620750 1204 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:41:08.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:08.623632 1204 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:41:08.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:08.625108 1204 hybrid_clock.cc:648] HybridClock initialized: now 1790646068625077 us; error 46 us; skew 500 ppm
01:41:08.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:08.625994 1204 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:41:08.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 01:41:08 dist-test-slave-rw6r krb5kdc[1114](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790646068, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:41:08.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:08.632272 1204 init.cc:377] Logged in from keytab as kudu/127.1.19.125@KRBTEST.COM (short username kudu)
01:41:08.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:08.638051 1204 webserver.cc:460] Webserver started at http://127.1.19.125:41349/ using document root <none> and password file <none>
01:41:08.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:08.639022 1204 fs_manager.cc:359] Metadata directory not provided
01:41:08.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:08.639256 1204 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:41:08.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:08.639854 1204 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:41:08.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:08.644948 1204 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-1/data/instance:
01:41:08.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fce2846bed844761afbf9124220f6e42"
01:41:08.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:41:08 on dist-test-slave-rw6r"
01:41:08.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:08.646257 1204 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-1/wal/instance:
01:41:08.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fce2846bed844761afbf9124220f6e42"
01:41:08.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:41:08 on dist-test-slave-rw6r"
01:41:08.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:08.654106 1204 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.002s sys 0.008s
01:41:08.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:08.659544 1224 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:41:08.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:08.660616 1204 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.001s sys 0.002s
01:41:08.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:08.660990 1204 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-1/data,/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-1/wal
01:41:08.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fce2846bed844761afbf9124220f6e42"
01:41:08.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:41:08 on dist-test-slave-rw6r"
01:41:08.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:08.661340 1204 fs_report.cc:383] FS layout report
01:41:08.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:41:08.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-1/wal
01:41:08.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-1/wal
01:41:08.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-1/data/data
01:41:08.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:41:08.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:41:08.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:41:08.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:41:08.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:41:08.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:41:08.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:41:08.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:41:08.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:41:08.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:41:08.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:41:08.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:08.730391 1204 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:41:08.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:08.736652 1204 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:41:08.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:08.737211 1204 kserver.cc:163] Server-wide thread pool size limit: 3276
01:41:08.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:08.815091 1204 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.125:39053
01:41:08.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:08.815208 1275 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.125:39053 every 8 connection(s)
01:41:08.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:08.817821 1204 server_base.cc:1166] Dumped server information to /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-1/data/info.pb
01:41:08.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:08.824033 1276 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:41:08.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:08.827129 1101 external_mini_cluster.cc:1388] Started /tmp/dist-test-task5IFP5y/build/tsan/bin/kudu as pid 1204
01:41:08.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:08.827590 1101 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-1/wal/instance
01:41:08.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:08.839283 1276 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.126" port: 40005 } has no permanent_uuid. Determining permanent_uuid...
01:41:08.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.19.124@KRBTEST.COM; defaulting to no policy
01:41:08.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:41:08.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.19.124@KRBTEST.COM" created.
01:41:08.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 01:41:08 dist-test-slave-rw6r krb5kdc[1114](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790646068, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.125@KRBTEST.COM for kudu/127.1.19.126@KRBTEST.COM
01:41:08.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:41:08.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/krb5kdc/kudu_127.1.19.124.keytab.
01:41:08.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/krb5kdc/kudu_127.1.19.124.keytab.
01:41:08.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:08.878398 1101 external_mini_cluster.cc:1326] Running /tmp/dist-test-task5IFP5y/build/tsan/bin/kudu
01:41:08.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5IFP5y/build/tsan/bin/kudu
01:41:08.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-2/wal
01:41:08.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-2/data
01:41:08.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:41:08.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:41:08.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:41:08.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:41:08.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:41:08.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:41:08.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-2/logs
01:41:08.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-2/data/info.pb
01:41:08.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:41:08.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:41:08.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:41:08.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:41:08.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:41:08.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:41:08.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:41:08.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:41:08.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:41:08.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:41:08.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:41:08.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:41:08.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.124:44787
01:41:08.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.124
01:41:08.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:41:08.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/krb5kdc/kudu_127.1.19.124.keytab
01:41:08.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.124
01:41:08.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:41:08.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:41:08.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:41:08.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.84:38321
01:41:08.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:41:08.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:41:08.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:41:08.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:41:08.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.126:40005,127.1.19.125:39053,127.1.19.124:44787
01:41:08.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:41:08.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45993/jwks.json
01:41:08.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:41:08.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:08.888240 1276 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.125" port: 39053 } has no permanent_uuid. Determining permanent_uuid...
01:41:08.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 01:41:08 dist-test-slave-rw6r krb5kdc[1114](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790646068, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.125@KRBTEST.COM for kudu/127.1.19.125@KRBTEST.COM
01:41:08.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:08.916024 1276 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.124" port: 44787 } has no permanent_uuid. Determining permanent_uuid...
01:41:08.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:08.918632 1228 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.124:44787: connect: Connection refused (error 111)
01:41:08.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:08.924217 1276 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.124:44787: Network error: Client connection negotiation failed: client connection to 127.1.19.124:44787: connect: Connection refused (error 111)
01:41:08.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:08.973682 1276 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.124" port: 44787 } attempt: 1
01:41:08.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:08.977779 1276 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.124:44787: Network error: Client connection negotiation failed: client connection to 127.1.19.124:44787: connect: Connection refused (error 111)
01:41:09.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:09.046247 1276 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.124" port: 44787 } attempt: 2
01:41:09.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:09.050140 1276 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.124:44787: Network error: Client connection negotiation failed: client connection to 127.1.19.124:44787: connect: Connection refused (error 111)
01:41:09.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:09.141513 1276 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.124" port: 44787 } attempt: 3
01:41:09.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:09.145809 1276 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.124:44787: Network error: Client connection negotiation failed: client connection to 127.1.19.124:44787: connect: Connection refused (error 111)
01:41:09.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:09.211151 1198 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.125" port: 39053 } attempt: 7
01:41:09.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:09.216220 1282 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:41:09.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:09.216832 1282 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:41:09.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:09.217857 1282 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:41:09.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 01:41:09 dist-test-slave-rw6r krb5kdc[1114](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790646066, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.126@KRBTEST.COM for kudu/127.1.19.125@KRBTEST.COM
01:41:09.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:09.240477 1198 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.124" port: 44787 } has no permanent_uuid. Determining permanent_uuid...
01:41:09.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:09.246258 1198 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.124:44787: Network error: Client connection negotiation failed: client connection to 127.1.19.124:44787: connect: Connection refused (error 111)
01:41:09.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:09.256064 1282 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:41:09.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:09.256274 1282 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:41:09.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:09.256522 1282 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:41:09.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:09.256706 1282 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:41:09.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:09.256896 1282 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:41:09.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:09.257103 1282 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:41:09.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:09.257234 1282 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45993/jwks.json
01:41:09.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:09.289258 1276 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.124" port: 44787 } attempt: 4
01:41:09.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:09.293046 1276 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.124:44787: Network error: Client connection negotiation failed: client connection to 127.1.19.124:44787: connect: Connection refused (error 111)
01:41:09.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:09.296113 1282 master_runner.cc:386] Master server non-default flags:
01:41:09.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:41:09.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.84:38321
01:41:09.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:41:09.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:41:09.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-2/data
01:41:09.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-2/wal
01:41:09.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:41:09.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:41:09.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.126:40005,127.1.19.125:39053,127.1.19.124:44787
01:41:09.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/krb5kdc/kudu_127.1.19.124.keytab
01:41:09.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.124
01:41:09.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:41:09.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:41:09.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:41:09.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.124:44787
01:41:09.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:41:09.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:41:09.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:41:09.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45993/jwks.json
01:41:09.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:41:09.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:41:09.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:41:09.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:41:09.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:41:09.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-2/data/info.pb
01:41:09.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.124
01:41:09.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:41:09.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:41:09.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:41:09.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:41:09.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:41:09.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:41:09.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-2/logs
01:41:09.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:41:09.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:41:09.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:41:09.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:41:09.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:41:09.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:41:09.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:41:09.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 00:23:04 UTC on ccycloud.quasar-noncbr.root.comops.site
01:41:09.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4678933
01:41:09.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:41:09.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:09.297508 1282 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:41:09.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:09.298656 1282 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:41:09.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:09.305097 1198 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.124" port: 44787 } attempt: 1
01:41:09.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:09.309785 1198 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.124:44787: Network error: Client connection negotiation failed: client connection to 127.1.19.124:44787: connect: Connection refused (error 111)
01:41:09.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:09.318397 1293 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:41:09.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:09.391311 1198 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.124" port: 44787 } attempt: 2
01:41:09.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:09.396076 1198 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.124:44787: Network error: Client connection negotiation failed: client connection to 127.1.19.124:44787: connect: Connection refused (error 111)
01:41:09.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:09.481475 1198 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.124" port: 44787 } attempt: 3
01:41:09.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:09.486042 1198 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.124:44787: Network error: Client connection negotiation failed: client connection to 127.1.19.124:44787: connect: Connection refused (error 111)
01:41:09.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:09.592451 1276 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.124" port: 44787 } attempt: 5
01:41:09.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:09.596603 1276 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.124:44787: Network error: Client connection negotiation failed: client connection to 127.1.19.124:44787: connect: Connection refused (error 111)
01:41:09.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:09.626434 1198 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.124" port: 44787 } attempt: 4
01:41:09.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:09.631076 1198 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.124:44787: Network error: Client connection negotiation failed: client connection to 127.1.19.124:44787: connect: Connection refused (error 111)
01:41:09.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:09.914499 1198 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.124" port: 44787 } attempt: 5
01:41:09.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:09.919027 1198 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.124:44787: Network error: Client connection negotiation failed: client connection to 127.1.19.124:44787: connect: Connection refused (error 111)
01:41:10.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:10.144564 1276 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.124" port: 44787 } attempt: 6
01:41:10.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:10.153236 1276 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.124:44787: Network error: Client connection negotiation failed: client connection to 127.1.19.124:44787: connect: Connection refused (error 111)
01:41:10.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:10.471444 1198 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.124" port: 44787 } attempt: 6
01:41:10.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:10.478407 1198 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.124:44787: Network error: Client connection negotiation failed: client connection to 127.1.19.124:44787: connect: Connection refused (error 111)
01:41:10.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:09.318424 1295 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:41:10.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:09.321203 1282 server_base.cc:1034] running on GCE node
01:41:10.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:09.319424 1292 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:41:10.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:10.600034 1282 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:41:10.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:10.602861 1282 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:41:10.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:10.604238 1282 hybrid_clock.cc:648] HybridClock initialized: now 1790646070604202 us; error 39 us; skew 500 ppm
01:41:10.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:10.605139 1282 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:41:10.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 01:41:10 dist-test-slave-rw6r krb5kdc[1114](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790646070, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:41:10.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:10.612464 1282 init.cc:377] Logged in from keytab as kudu/127.1.19.124@KRBTEST.COM (short username kudu)
01:41:10.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:10.618351 1282 webserver.cc:460] Webserver started at http://127.1.19.124:45195/ using document root <none> and password file <none>
01:41:10.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:10.619550 1282 fs_manager.cc:359] Metadata directory not provided
01:41:10.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:10.619846 1282 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:41:10.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:10.620375 1282 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:41:10.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:10.625294 1282 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-2/data/instance:
01:41:10.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a8e7a7b6c3bc4ffebab8c3c5428bfaf4"
01:41:10.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:41:10 on dist-test-slave-rw6r"
01:41:10.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:10.626454 1282 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-2/wal/instance:
01:41:10.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a8e7a7b6c3bc4ffebab8c3c5428bfaf4"
01:41:10.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:41:10 on dist-test-slave-rw6r"
01:41:10.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:10.633960 1282 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.004s
01:41:10.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:10.639724 1310 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:41:10.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:10.641314 1282 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.001s sys 0.002s
01:41:10.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:10.641685 1282 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-2/data,/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-2/wal
01:41:10.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a8e7a7b6c3bc4ffebab8c3c5428bfaf4"
01:41:10.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:41:10 on dist-test-slave-rw6r"
01:41:10.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:10.642108 1282 fs_report.cc:383] FS layout report
01:41:10.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:41:10.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-2/wal
01:41:10.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-2/wal
01:41:10.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-2/data/data
01:41:10.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:41:10.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:41:10.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:41:10.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:41:10.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:41:10.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:41:10.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:41:10.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:41:10.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:41:10.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:41:10.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:41:10.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:10.678119 1282 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:41:10.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:10.684659 1282 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:41:10.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:10.685186 1282 kserver.cc:163] Server-wide thread pool size limit: 3276
01:41:10.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:10.769601 1282 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.124:44787
01:41:10.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:10.769682 1361 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.124:44787 every 8 connection(s)
01:41:10.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:10.772601 1282 server_base.cc:1166] Dumped server information to /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-2/data/info.pb
01:41:10.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:10.778973 1362 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:41:10.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:10.782816 1101 external_mini_cluster.cc:1388] Started /tmp/dist-test-task5IFP5y/build/tsan/bin/kudu as pid 1282
01:41:10.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:10.783318 1101 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-2/wal/instance
01:41:10.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:10.796473 1362 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.126" port: 40005 } has no permanent_uuid. Determining permanent_uuid...
01:41:10.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 01:41:10 dist-test-slave-rw6r krb5kdc[1114](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790646070, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.124@KRBTEST.COM for kudu/127.1.19.126@KRBTEST.COM
01:41:10.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 01:41:10 dist-test-slave-rw6r krb5kdc[1114](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790646064, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.19.124@KRBTEST.COM
01:41:10.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:10.836372 1362 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.125" port: 39053 } has no permanent_uuid. Determining permanent_uuid...
01:41:10.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 01:41:10 dist-test-slave-rw6r krb5kdc[1114](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790646070, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.124@KRBTEST.COM for kudu/127.1.19.125@KRBTEST.COM
01:41:10.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 01:41:10 dist-test-slave-rw6r krb5kdc[1114](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790646064, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.19.125@KRBTEST.COM
01:41:10.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:10.868232 1362 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.124" port: 44787 } has no permanent_uuid. Determining permanent_uuid...
01:41:10.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 01:41:10 dist-test-slave-rw6r krb5kdc[1114](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790646070, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.124@KRBTEST.COM for kudu/127.1.19.124@KRBTEST.COM
01:41:10.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 01:41:10 dist-test-slave-rw6r krb5kdc[1114](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790646064, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.19.126@KRBTEST.COM
01:41:10.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:10.902103 1362 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4: Bootstrap starting.
01:41:10.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:10.906738 1101 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
01:41:10.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:10.907445 1362 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4: Neither blocks nor log segments found. Creating new log.
01:41:10.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:10.909232 1362 log.cc:824] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4: Log is configured to *not* fsync() on all Append() calls
01:41:10.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:10.913777 1362 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4: No bootstrap required, opened a new log
01:41:10.914 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:41:10.916 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
01:41:10.928 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
01:41:10.929 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
01:41:10.932 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.1.19.126"
port: 40005
}
bound_http_address {
host: "127.1.19.126"
port: 42113
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.1.19.125"
port: 39053
}
bound_http_address {
host: "127.1.19.125"
port: 41349
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.1.19.124"
port: 44787
}
bound_http_address {
host: "127.1.19.124"
port: 45195
}
}
}
01:41:10.933 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
01:41:10.933 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:41:10.933 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
01:41:10.935 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
01:41:10.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:10.949497 1362 raft_consensus.cc:348] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "85a011fbef4b438c84f15e8e42b174c8" member_type: VOTER last_known_addr { host: "127.1.19.126" port: 40005 } } peers { permanent_uuid: "fce2846bed844761afbf9124220f6e42" member_type: VOTER last_known_addr { host: "127.1.19.125" port: 39053 } } peers { permanent_uuid: "a8e7a7b6c3bc4ffebab8c3c5428bfaf4" member_type: VOTER last_known_addr { host: "127.1.19.124" port: 44787 } }
01:41:10.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:10.950387 1362 raft_consensus.cc:374] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
01:41:10.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:10.950665 1362 raft_consensus.cc:729] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a8e7a7b6c3bc4ffebab8c3c5428bfaf4, State: Initialized, Role: FOLLOWER
01:41:10.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:10.951562 1362 consensus_queue.cc:260] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "85a011fbef4b438c84f15e8e42b174c8" member_type: VOTER last_known_addr { host: "127.1.19.126" port: 40005 } } peers { permanent_uuid: "fce2846bed844761afbf9124220f6e42" member_type: VOTER last_known_addr { host: "127.1.19.125" port: 39053 } } peers { permanent_uuid: "a8e7a7b6c3bc4ffebab8c3c5428bfaf4" member_type: VOTER last_known_addr { host: "127.1.19.124" port: 44787 } }
01:41:10.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:10.955863 1372 sys_catalog.cc:455] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "85a011fbef4b438c84f15e8e42b174c8" member_type: VOTER last_known_addr { host: "127.1.19.126" port: 40005 } } peers { permanent_uuid: "fce2846bed844761afbf9124220f6e42" member_type: VOTER last_known_addr { host: "127.1.19.125" port: 39053 } } peers { permanent_uuid: "a8e7a7b6c3bc4ffebab8c3c5428bfaf4" member_type: VOTER last_known_addr { host: "127.1.19.124" port: 44787 } } }
01:41:10.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:10.956826 1372 sys_catalog.cc:458] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [sys.catalog]: This master's current role is: FOLLOWER
01:41:10.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:10.957904 1362 sys_catalog.cc:564] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [sys.catalog]: configured and running, proceeding with master startup.
01:41:10.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 01:41:10 dist-test-slave-rw6r krb5kdc[1114](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790646070, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:41:10.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
01:41:10.978 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:41:10.981 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/krb5kdc/krb5cc
01:41:10.989 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
01:41:10.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:10.994791 1384 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:41:10.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:10.995363 1384 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:41:10.996 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/krb5kdc/krb5cc
01:41:11.000 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
01:41:11.005 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
01:41:11.007 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
01:41:11.008 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
01:41:11.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:10.987565 1282 master_runner.cc:186] Error getting master registration for 127.1.19.126:40005: OK, instance_id { permanent_uuid: "85a011fbef4b438c84f15e8e42b174c8" instance_seqno: 1790646066824101 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
01:41:11.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:11.018230 1282 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:41:11.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 01:41:11 dist-test-slave-rw6r krb5kdc[1114](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790646071, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:41:11.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
01:41:11.044 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:41:11.045 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/krb5kdc/krb5cc
01:41:11.046 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
01:41:11.047 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.19.125:39053
01:41:11.055 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
01:41:11.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:11.056572 1101 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task5IFP5y/build/tsan/bin/kudu with pid 1204
01:41:11.080 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:41:11.080 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.19.126:40005
01:41:11.080 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
01:41:11.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:11.081034 1101 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task5IFP5y/build/tsan/bin/kudu with pid 1127
01:41:11.107 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:41:11.107 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.19.124:44787
01:41:11.108 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
01:41:11.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:11.108530 1101 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task5IFP5y/build/tsan/bin/kudu with pid 1282
01:41:11.130 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:41:11.130 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.19.125:39053
01:41:11.132 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
01:41:11.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:11.134042 1101 external_mini_cluster.cc:1326] Running /tmp/dist-test-task5IFP5y/build/tsan/bin/kudu
01:41:11.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5IFP5y/build/tsan/bin/kudu
01:41:11.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-1/wal
01:41:11.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-1/data
01:41:11.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:41:11.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:41:11.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:41:11.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:41:11.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:41:11.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:41:11.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-1/logs
01:41:11.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-1/data/info.pb
01:41:11.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:41:11.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:41:11.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:41:11.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:41:11.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:41:11.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:41:11.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:41:11.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:41:11.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:41:11.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:41:11.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:41:11.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:41:11.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.125:39053
01:41:11.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.125
01:41:11.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41349
01:41:11.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/krb5kdc/kudu_127.1.19.125.keytab
01:41:11.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.125
01:41:11.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:41:11.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:41:11.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:41:11.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.84:38321
01:41:11.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:41:11.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:41:11.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:41:11.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:41:11.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.126:40005,127.1.19.125:39053,127.1.19.124:44787
01:41:11.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:41:11.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45993/jwks.json
01:41:11.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:41:11.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:11.450582 1388 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:41:11.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:11.451169 1388 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:41:11.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:11.451843 1388 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:41:11.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:11.482574 1388 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:41:11.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:11.482815 1388 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:41:11.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:11.483058 1388 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:41:11.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:11.483244 1388 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:41:11.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:11.483421 1388 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:41:11.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:11.483595 1388 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:41:11.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:11.483832 1388 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45993/jwks.json
01:41:11.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:11.519330 1388 master_runner.cc:386] Master server non-default flags:
01:41:11.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:41:11.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.84:38321
01:41:11.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:41:11.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:41:11.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-1/data
01:41:11.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-1/wal
01:41:11.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:41:11.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:41:11.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.126:40005,127.1.19.125:39053,127.1.19.124:44787
01:41:11.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/krb5kdc/kudu_127.1.19.125.keytab
01:41:11.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.125
01:41:11.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:41:11.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:41:11.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:41:11.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.125:39053
01:41:11.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:41:11.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:41:11.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:41:11.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45993/jwks.json
01:41:11.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:41:11.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:41:11.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:41:11.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:41:11.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:41:11.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-1/data/info.pb
01:41:11.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.125
01:41:11.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41349
01:41:11.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:41:11.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:41:11.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:41:11.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:41:11.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:41:11.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-1/logs
01:41:11.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:41:11.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:41:11.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:41:11.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:41:11.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:41:11.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:41:11.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:41:11.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 00:23:04 UTC on ccycloud.quasar-noncbr.root.comops.site
01:41:11.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4678933
01:41:11.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:41:11.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:11.520462 1388 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:41:11.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:11.521384 1388 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:41:11.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:11.539141 1396 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:41:12.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:11.540222 1395 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:41:12.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:11.539556 1398 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:41:12.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:12.770627 1397 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
01:41:12.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:12.770782 1388 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
01:41:12.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:12.774423 1388 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:41:12.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:12.777004 1388 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:41:12.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:12.778349 1388 hybrid_clock.cc:648] HybridClock initialized: now 1790646072778315 us; error 51 us; skew 500 ppm
01:41:12.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:12.779111 1388 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:41:12.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 01:41:12 dist-test-slave-rw6r krb5kdc[1114](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790646072, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:41:12.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:12.784924 1388 init.cc:377] Logged in from keytab as kudu/127.1.19.125@KRBTEST.COM (short username kudu)
01:41:12.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:12.789594 1388 webserver.cc:460] Webserver started at http://127.1.19.125:41349/ using document root <none> and password file <none>
01:41:12.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:12.790452 1388 fs_manager.cc:359] Metadata directory not provided
01:41:12.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:12.790653 1388 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:41:12.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:12.798233 1388 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.005s sys 0.000s
01:41:12.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:12.802564 1407 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:41:12.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:12.803531 1388 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
01:41:12.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:12.803906 1388 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-1/data,/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-1/wal
01:41:12.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fce2846bed844761afbf9124220f6e42"
01:41:12.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:41:08 on dist-test-slave-rw6r"
01:41:12.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:12.806180 1388 fs_report.cc:383] FS layout report
01:41:12.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:41:12.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-1/wal
01:41:12.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-1/wal
01:41:12.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-1/data/data
01:41:12.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:41:12.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:41:12.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:41:12.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:41:12.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:41:12.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:41:12.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:41:12.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:41:12.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:41:12.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:41:12.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:41:12.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:12.886238 1388 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:41:12.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:12.892416 1388 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:41:12.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:12.892949 1388 kserver.cc:163] Server-wide thread pool size limit: 3276
01:41:12.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:12.966137 1388 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.125:39053
01:41:12.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:12.966213 1458 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.125:39053 every 8 connection(s)
01:41:12.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:12.969383 1388 server_base.cc:1166] Dumped server information to /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-1/data/info.pb
01:41:12.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:12.979530 1101 external_mini_cluster.cc:1388] Started /tmp/dist-test-task5IFP5y/build/tsan/bin/kudu as pid 1388
01:41:12.980 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:41:12.980 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.19.126:40005
01:41:12.980 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
01:41:12.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:12.980851 1459 sys_catalog.cc:263] Verifying existing consensus state
01:41:12.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260929 01:41:12.981805 1459 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
01:41:12.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:12.982141 1101 external_mini_cluster.cc:1326] Running /tmp/dist-test-task5IFP5y/build/tsan/bin/kudu
01:41:12.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5IFP5y/build/tsan/bin/kudu
01:41:12.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-0/wal
01:41:12.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-0/data
01:41:12.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:41:12.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:41:12.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:41:12.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:41:12.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:41:12.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:41:12.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-0/logs
01:41:12.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-0/data/info.pb
01:41:12.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:41:12.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:41:12.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:41:12.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:41:12.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:41:12.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:41:12.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:41:12.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:41:12.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:41:12.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:41:12.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:41:12.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:41:12.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.126:40005
01:41:12.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.126
01:41:12.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42113
01:41:12.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/krb5kdc/kudu_127.1.19.126.keytab
01:41:12.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.126
01:41:12.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:41:12.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:41:12.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:41:12.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.84:38321
01:41:12.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:41:12.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:41:12.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:41:12.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:41:12.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.126:40005,127.1.19.125:39053,127.1.19.124:44787
01:41:12.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:41:12.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45993/jwks.json
01:41:12.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:41:12.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:12.982342 1388 master.cc:537] Master@127.1.19.125:39053 shutting down...
01:41:13.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:13.048688 1388 master.cc:552] Master@127.1.19.125:39053 shutdown complete.
01:41:13.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
01:41:13.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:13.310604 1461 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:41:13.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:13.311288 1461 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:41:13.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:13.312083 1461 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:41:13.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:13.343459 1461 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:41:13.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:13.343775 1461 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:41:13.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:13.344200 1461 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:41:13.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:13.344563 1461 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:41:13.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:13.344884 1461 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:41:13.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:13.345228 1461 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:41:13.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:13.345466 1461 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45993/jwks.json
01:41:13.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:13.383908 1461 master_runner.cc:386] Master server non-default flags:
01:41:13.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:41:13.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.84:38321
01:41:13.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:41:13.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:41:13.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-0/data
01:41:13.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-0/wal
01:41:13.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:41:13.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:41:13.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.126:40005,127.1.19.125:39053,127.1.19.124:44787
01:41:13.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/krb5kdc/kudu_127.1.19.126.keytab
01:41:13.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.126
01:41:13.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:41:13.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:41:13.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:41:13.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.126:40005
01:41:13.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:41:13.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:41:13.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:41:13.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45993/jwks.json
01:41:13.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:41:13.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:41:13.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:41:13.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:41:13.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:41:13.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-0/data/info.pb
01:41:13.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.126
01:41:13.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42113
01:41:13.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:41:13.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:41:13.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:41:13.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:41:13.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:41:13.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-0/logs
01:41:13.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:41:13.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:41:13.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:41:13.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:41:13.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:41:13.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:41:13.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:41:13.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 00:23:04 UTC on ccycloud.quasar-noncbr.root.comops.site
01:41:13.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4678933
01:41:13.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:41:13.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:13.385622 1461 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:41:13.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:13.387120 1461 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:41:13.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:13.403292 1467 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:41:14.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:13.403551 1470 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:41:14.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:13.404594 1468 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:41:14.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:14.625530 1469 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
01:41:14.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:14.625617 1461 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
01:41:14.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:14.629351 1461 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:41:14.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:14.632072 1461 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:41:14.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:14.633453 1461 hybrid_clock.cc:648] HybridClock initialized: now 1790646074633390 us; error 62 us; skew 500 ppm
01:41:14.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:14.634284 1461 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:41:14.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 01:41:14 dist-test-slave-rw6r krb5kdc[1114](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790646074, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:41:14.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:14.644603 1461 init.cc:377] Logged in from keytab as kudu/127.1.19.126@KRBTEST.COM (short username kudu)
01:41:14.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:14.649915 1461 webserver.cc:460] Webserver started at http://127.1.19.126:42113/ using document root <none> and password file <none>
01:41:14.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:14.650978 1461 fs_manager.cc:359] Metadata directory not provided
01:41:14.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:14.651218 1461 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:41:14.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:14.659389 1461 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.002s sys 0.005s
01:41:14.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:14.664157 1480 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:41:14.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:14.665411 1461 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
01:41:14.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:14.665767 1461 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-0/data,/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-0/wal
01:41:14.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "85a011fbef4b438c84f15e8e42b174c8"
01:41:14.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:41:06 on dist-test-slave-rw6r"
01:41:14.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:14.668247 1461 fs_report.cc:383] FS layout report
01:41:14.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:41:14.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-0/wal
01:41:14.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-0/wal
01:41:14.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-0/data/data
01:41:14.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:41:14.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:41:14.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:41:14.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:41:14.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:41:14.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:41:14.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:41:14.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:41:14.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:41:14.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:41:14.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:41:14.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:14.717480 1461 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:41:14.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:14.722931 1461 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:41:14.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:14.723469 1461 kserver.cc:163] Server-wide thread pool size limit: 3276
01:41:14.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:14.794564 1461 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.126:40005
01:41:14.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:14.794701 1531 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.126:40005 every 8 connection(s)
01:41:14.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:14.797294 1461 server_base.cc:1166] Dumped server information to /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-0/data/info.pb
01:41:14.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:14.800529 1101 external_mini_cluster.cc:1388] Started /tmp/dist-test-task5IFP5y/build/tsan/bin/kudu as pid 1461
01:41:14.802 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:41:14.802 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.19.124:44787
01:41:14.802 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
01:41:14.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:14.803798 1101 external_mini_cluster.cc:1326] Running /tmp/dist-test-task5IFP5y/build/tsan/bin/kudu
01:41:14.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5IFP5y/build/tsan/bin/kudu
01:41:14.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-2/wal
01:41:14.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-2/data
01:41:14.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:41:14.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:41:14.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:41:14.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:41:14.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:41:14.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:41:14.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-2/logs
01:41:14.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-2/data/info.pb
01:41:14.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:41:14.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:41:14.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:41:14.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:41:14.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:41:14.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:41:14.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:41:14.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:41:14.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:41:14.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:41:14.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:41:14.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:41:14.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.124:44787
01:41:14.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.124
01:41:14.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45195
01:41:14.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/krb5kdc/kudu_127.1.19.124.keytab
01:41:14.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.124
01:41:14.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:41:14.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:41:14.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:41:14.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.84:38321
01:41:14.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:41:14.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:41:14.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:41:14.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:41:14.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.126:40005,127.1.19.125:39053,127.1.19.124:44787
01:41:14.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:41:14.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45993/jwks.json
01:41:14.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:41:14.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:14.808424 1532 sys_catalog.cc:263] Verifying existing consensus state
01:41:14.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260929 01:41:14.809417 1532 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
01:41:14.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:14.809881 1461 master.cc:537] Master@127.1.19.126:40005 shutting down...
01:41:14.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:14.870386 1461 master.cc:552] Master@127.1.19.126:40005 shutdown complete.
01:41:14.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
01:41:15.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:15.131956 1534 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:41:15.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:15.132505 1534 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:41:15.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:15.133085 1534 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:41:15.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:15.167523 1534 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:41:15.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:15.167824 1534 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:41:15.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:15.168107 1534 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:41:15.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:15.168303 1534 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:41:15.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:15.168488 1534 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:41:15.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:15.168671 1534 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:41:15.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:15.168804 1534 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45993/jwks.json
01:41:15.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:15.207675 1534 master_runner.cc:386] Master server non-default flags:
01:41:15.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:41:15.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.84:38321
01:41:15.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:41:15.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:41:15.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-2/data
01:41:15.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-2/wal
01:41:15.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:41:15.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:41:15.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.126:40005,127.1.19.125:39053,127.1.19.124:44787
01:41:15.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/krb5kdc/kudu_127.1.19.124.keytab
01:41:15.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.124
01:41:15.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:41:15.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:41:15.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:41:15.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.124:44787
01:41:15.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:41:15.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:41:15.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:41:15.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45993/jwks.json
01:41:15.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:41:15.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:41:15.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:41:15.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:41:15.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:41:15.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-2/data/info.pb
01:41:15.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.124
01:41:15.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45195
01:41:15.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:41:15.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:41:15.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:41:15.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:41:15.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:41:15.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-2/logs
01:41:15.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:41:15.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:41:15.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:41:15.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:41:15.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:41:15.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:41:15.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:41:15.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 00:23:04 UTC on ccycloud.quasar-noncbr.root.comops.site
01:41:15.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4678933
01:41:15.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:41:15.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:15.208916 1534 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:41:15.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:15.209940 1534 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:41:15.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:15.229647 1540 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:41:16.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:15.229714 1541 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:41:16.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:15.232807 1543 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:41:16.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:16.557703 1542 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1326 milliseconds
01:41:16.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:16.557823 1534 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
01:41:16.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:16.559226 1534 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:41:16.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:16.562486 1534 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:41:16.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:16.563925 1534 hybrid_clock.cc:648] HybridClock initialized: now 1790646076563881 us; error 63 us; skew 500 ppm
01:41:16.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:16.564877 1534 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:41:16.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 01:41:16 dist-test-slave-rw6r krb5kdc[1114](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790646076, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:41:16.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:16.571101 1534 init.cc:377] Logged in from keytab as kudu/127.1.19.124@KRBTEST.COM (short username kudu)
01:41:16.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:16.576711 1534 webserver.cc:460] Webserver started at http://127.1.19.124:45195/ using document root <none> and password file <none>
01:41:16.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:16.577739 1534 fs_manager.cc:359] Metadata directory not provided
01:41:16.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:16.578018 1534 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:41:16.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:16.586477 1534 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.002s sys 0.003s
01:41:16.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:16.590972 1552 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:41:16.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:16.592074 1534 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
01:41:16.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:16.592442 1534 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-2/data,/tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-2/wal
01:41:16.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a8e7a7b6c3bc4ffebab8c3c5428bfaf4"
01:41:16.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:41:10 on dist-test-slave-rw6r"
01:41:16.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:16.594743 1534 fs_report.cc:383] FS layout report
01:41:16.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:41:16.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-2/wal
01:41:16.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-2/wal
01:41:16.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-2/data/data
01:41:16.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:41:16.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:41:16.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:41:16.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:41:16.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:41:16.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:41:16.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:41:16.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:41:16.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:41:16.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:41:16.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:41:16.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:16.633347 1534 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:41:16.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:16.639204 1534 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:41:16.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:16.639781 1534 kserver.cc:163] Server-wide thread pool size limit: 3276
01:41:16.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:16.716833 1534 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.124:44787
01:41:16.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:16.716962 1603 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.124:44787 every 8 connection(s)
01:41:16.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:16.720575 1534 server_base.cc:1166] Dumped server information to /tmp/dist-test-task5IFP5y/test-tmp/mini-kudu-cluster9182396620793234843/master-2/data/info.pb
01:41:16.722 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:41:16.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:16.721889 1101 external_mini_cluster.cc:1388] Started /tmp/dist-test-task5IFP5y/build/tsan/bin/kudu as pid 1534
01:41:16.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:16.733374 1604 sys_catalog.cc:263] Verifying existing consensus state
01:41:16.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:16.740355 1604 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4: Bootstrap starting.
01:41:16.773 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.19.125:39053(127.1.19.125:39053)] connecting to peer
01:41:16.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:16.783377 1604 log.cc:824] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4: Log is configured to *not* fsync() on all Append() calls
01:41:16.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:16.789302 1604 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
01:41:16.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:16.790228 1604 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4: Bootstrap complete.
01:41:16.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:16.823510 1604 raft_consensus.cc:348] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "85a011fbef4b438c84f15e8e42b174c8" member_type: VOTER last_known_addr { host: "127.1.19.126" port: 40005 } } peers { permanent_uuid: "fce2846bed844761afbf9124220f6e42" member_type: VOTER last_known_addr { host: "127.1.19.125" port: 39053 } } peers { permanent_uuid: "a8e7a7b6c3bc4ffebab8c3c5428bfaf4" member_type: VOTER last_known_addr { host: "127.1.19.124" port: 44787 } }
01:41:16.827 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.19.126:40005(127.1.19.126:40005)] connecting to peer
01:41:16.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:16.826924 1604 raft_consensus.cc:374] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
01:41:16.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:16.827247 1604 raft_consensus.cc:729] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a8e7a7b6c3bc4ffebab8c3c5428bfaf4, State: Initialized, Role: FOLLOWER
01:41:16.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:16.828478 1604 consensus_queue.cc:260] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "85a011fbef4b438c84f15e8e42b174c8" member_type: VOTER last_known_addr { host: "127.1.19.126" port: 40005 } } peers { permanent_uuid: "fce2846bed844761afbf9124220f6e42" member_type: VOTER last_known_addr { host: "127.1.19.125" port: 39053 } } peers { permanent_uuid: "a8e7a7b6c3bc4ffebab8c3c5428bfaf4" member_type: VOTER last_known_addr { host: "127.1.19.124" port: 44787 } }
01:41:16.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:16.831490 1610 sys_catalog.cc:455] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "85a011fbef4b438c84f15e8e42b174c8" member_type: VOTER last_known_addr { host: "127.1.19.126" port: 40005 } } peers { permanent_uuid: "fce2846bed844761afbf9124220f6e42" member_type: VOTER last_known_addr { host: "127.1.19.125" port: 39053 } } peers { permanent_uuid: "a8e7a7b6c3bc4ffebab8c3c5428bfaf4" member_type: VOTER last_known_addr { host: "127.1.19.124" port: 44787 } } }
01:41:16.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:16.832531 1610 sys_catalog.cc:458] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [sys.catalog]: This master's current role is: FOLLOWER
01:41:16.833 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.19.124:44787(127.1.19.124:44787)] connecting to peer
01:41:16.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:16.833273 1604 sys_catalog.cc:564] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [sys.catalog]: configured and running, proceeding with master startup.
01:41:16.883 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:16.884 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.126:40005(127.1.19.126:40005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:16.882 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:16.887 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.125:39053(127.1.19.125:39053)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:16.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:16.888175 1621 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:41:16.888 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.126:40005: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:16.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:16.888593 1621 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:41:16.890 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.125:39053: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:16.890 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.1.19.124:44787(127.1.19.124:44787)] Successfully connected to peer
01:41:16.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:16.895080 1558 tablet.cc:2369] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4: Can't schedule compaction. Clean time has not been advanced past its initial value.
01:41:16.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:16.898517 1534 master_runner.cc:186] Error getting master registration for 127.1.19.126:40005: Network error: Client connection negotiation failed: client connection to 127.1.19.126:40005: connect: Connection refused (error 111),
01:41:16.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:16.928423 1534 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:41:16.980 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
01:41:16.989 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.1.19.124
01:41:17.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 01:41:17 dist-test-slave-rw6r krb5kdc[1114](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790646071, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.19.124@KRBTEST.COM
01:41:17.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:17.208907 1610 raft_consensus.cc:482] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:41:17.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:17.209393 1610 raft_consensus.cc:504] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "85a011fbef4b438c84f15e8e42b174c8" member_type: VOTER last_known_addr { host: "127.1.19.126" port: 40005 } } peers { permanent_uuid: "fce2846bed844761afbf9124220f6e42" member_type: VOTER last_known_addr { host: "127.1.19.125" port: 39053 } } peers { permanent_uuid: "a8e7a7b6c3bc4ffebab8c3c5428bfaf4" member_type: VOTER last_known_addr { host: "127.1.19.124" port: 44787 } }
01:41:17.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:17.217521 1555 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.126:40005: connect: Connection refused (error 111)
01:41:17.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:17.219312 1610 leader_election.cc:288] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 85a011fbef4b438c84f15e8e42b174c8 (127.1.19.126:40005), fce2846bed844761afbf9124220f6e42 (127.1.19.125:39053)
01:41:17.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:17.227283 1553 leader_election.cc:334] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer fce2846bed844761afbf9124220f6e42 (127.1.19.125:39053): Network error: Client connection negotiation failed: client connection to 127.1.19.125:39053: connect: Connection refused (error 111)
01:41:17.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:17.229282 1555 leader_election.cc:334] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 85a011fbef4b438c84f15e8e42b174c8 (127.1.19.126:40005): Network error: Client connection negotiation failed: client connection to 127.1.19.126:40005: connect: Connection refused (error 111)
01:41:17.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:17.229938 1555 leader_election.cc:302] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a8e7a7b6c3bc4ffebab8c3c5428bfaf4; no voters: 85a011fbef4b438c84f15e8e42b174c8, fce2846bed844761afbf9124220f6e42
01:41:17.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:17.230934 1610 raft_consensus.cc:2726] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
01:41:17.262 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x66ce4385, L:/127.0.0.1:56812 - R:/127.1.19.124:44787] using SASL/GSSAPI
01:41:17.281 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787; will retry
01:41:17.290 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.125:39053(127.1.19.125:39053)] connecting to peer
01:41:17.292 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.126:40005(127.1.19.126:40005)] connecting to peer
01:41:17.293 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:17.293 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.125:39053(127.1.19.125:39053)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:17.294 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.125:39053: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:17.296 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:17.296 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.126:40005(127.1.19.126:40005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:17.296 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.126:40005: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:17.301 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787; will retry
01:41:17.309 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.125:39053(127.1.19.125:39053)] connecting to peer
01:41:17.310 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.126:40005(127.1.19.126:40005)] connecting to peer
01:41:17.311 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:17.312 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.125:39053(127.1.19.125:39053)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:17.312 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.125:39053: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:17.317 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:17.317 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.126:40005(127.1.19.126:40005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:17.317 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.126:40005: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:17.318 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787; will retry
01:41:17.330 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.125:39053(127.1.19.125:39053)] connecting to peer
01:41:17.330 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.126:40005(127.1.19.126:40005)] connecting to peer
01:41:17.332 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:17.332 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.125:39053(127.1.19.125:39053)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:17.333 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.125:39053: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:17.336 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:17.336 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.126:40005(127.1.19.126:40005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:17.336 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.126:40005: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:17.338 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787; will retry
01:41:17.349 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.125:39053(127.1.19.125:39053)] connecting to peer
01:41:17.350 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.126:40005(127.1.19.126:40005)] connecting to peer
01:41:17.352 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:17.352 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.125:39053(127.1.19.125:39053)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:17.353 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.125:39053: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:17.357 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:17.357 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.126:40005(127.1.19.126:40005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:17.358 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.126:40005: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:17.359 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787; will retry
01:41:17.370 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.125:39053(127.1.19.125:39053)] connecting to peer
01:41:17.371 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.126:40005(127.1.19.126:40005)] connecting to peer
01:41:17.373 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:17.373 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.125:39053(127.1.19.125:39053)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:17.373 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.125:39053: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:17.377 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:17.377 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.126:40005(127.1.19.126:40005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:17.378 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.126:40005: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:17.380 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787; will retry
01:41:17.449 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.125:39053(127.1.19.125:39053)] connecting to peer
01:41:17.451 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.126:40005(127.1.19.126:40005)] connecting to peer
01:41:17.451 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:17.452 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.125:39053(127.1.19.125:39053)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:17.452 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.125:39053: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:17.456 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:17.456 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.126:40005(127.1.19.126:40005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:17.457 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.126:40005: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:17.459 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787; will retry
01:41:17.490 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.125:39053(127.1.19.125:39053)] connecting to peer
01:41:17.492 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.126:40005(127.1.19.126:40005)] connecting to peer
01:41:17.492 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:17.492 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.125:39053(127.1.19.125:39053)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:17.493 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.125:39053: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:17.495 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:17.495 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.126:40005(127.1.19.126:40005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:17.495 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.126:40005: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:17.498 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787; will retry
01:41:17.689 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.125:39053(127.1.19.125:39053)] connecting to peer
01:41:17.691 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.126:40005(127.1.19.126:40005)] connecting to peer
01:41:17.691 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:17.691 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.125:39053(127.1.19.125:39053)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:17.692 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.125:39053: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:17.697 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:17.697 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.126:40005(127.1.19.126:40005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:17.698 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.126:40005: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:17.699 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787; will retry
01:41:17.789 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.125:39053(127.1.19.125:39053)] connecting to peer
01:41:17.790 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.126:40005(127.1.19.126:40005)] connecting to peer
01:41:17.792 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:17.793 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.125:39053(127.1.19.125:39053)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:17.794 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.125:39053: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:17.794 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:17.795 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.126:40005(127.1.19.126:40005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:17.795 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.126:40005: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:17.797 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787; will retry
01:41:17.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:17.896904 1621 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:41:17.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:17.897245 1621 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:41:18.189 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.125:39053(127.1.19.125:39053)] connecting to peer
01:41:18.191 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:18.191 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.125:39053(127.1.19.125:39053)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:18.191 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.125:39053: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:18.192 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.126:40005(127.1.19.126:40005)] connecting to peer
01:41:18.194 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:18.194 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.126:40005(127.1.19.126:40005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:18.194 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.126:40005: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:18.198 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787; will retry
01:41:18.869 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.125:39053(127.1.19.125:39053)] connecting to peer
01:41:18.870 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.126:40005(127.1.19.126:40005)] connecting to peer
01:41:18.871 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:18.871 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.125:39053(127.1.19.125:39053)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:18.872 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.125:39053: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:18.872 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:18.872 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.126:40005(127.1.19.126:40005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:18.872 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.126:40005: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:18.876 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787; will retry
01:41:18.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:18.900069 1621 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:41:18.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:18.900354 1621 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:41:18.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:18.948896 1626 raft_consensus.cc:482] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:41:18.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:18.949344 1626 raft_consensus.cc:504] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "85a011fbef4b438c84f15e8e42b174c8" member_type: VOTER last_known_addr { host: "127.1.19.126" port: 40005 } } peers { permanent_uuid: "fce2846bed844761afbf9124220f6e42" member_type: VOTER last_known_addr { host: "127.1.19.125" port: 39053 } } peers { permanent_uuid: "a8e7a7b6c3bc4ffebab8c3c5428bfaf4" member_type: VOTER last_known_addr { host: "127.1.19.124" port: 44787 } }
01:41:18.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:18.950917 1626 leader_election.cc:288] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 85a011fbef4b438c84f15e8e42b174c8 (127.1.19.126:40005), fce2846bed844761afbf9124220f6e42 (127.1.19.125:39053)
01:41:18.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:18.956921 1555 leader_election.cc:334] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 85a011fbef4b438c84f15e8e42b174c8 (127.1.19.126:40005): Network error: Client connection negotiation failed: client connection to 127.1.19.126:40005: connect: Connection refused (error 111)
01:41:18.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:18.957305 1553 leader_election.cc:334] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer fce2846bed844761afbf9124220f6e42 (127.1.19.125:39053): Network error: Client connection negotiation failed: client connection to 127.1.19.125:39053: connect: Connection refused (error 111)
01:41:18.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:18.957608 1553 leader_election.cc:302] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a8e7a7b6c3bc4ffebab8c3c5428bfaf4; no voters: 85a011fbef4b438c84f15e8e42b174c8, fce2846bed844761afbf9124220f6e42
01:41:18.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:18.958134 1626 raft_consensus.cc:2726] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
01:41:19.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:19.903026 1621 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:41:19.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:19.903268 1621 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:41:20.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:20.906176 1621 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:41:20.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:20.906423 1621 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:41:21.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:21.670064 1634 raft_consensus.cc:482] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:41:21.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:21.670538 1634 raft_consensus.cc:504] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "85a011fbef4b438c84f15e8e42b174c8" member_type: VOTER last_known_addr { host: "127.1.19.126" port: 40005 } } peers { permanent_uuid: "fce2846bed844761afbf9124220f6e42" member_type: VOTER last_known_addr { host: "127.1.19.125" port: 39053 } } peers { permanent_uuid: "a8e7a7b6c3bc4ffebab8c3c5428bfaf4" member_type: VOTER last_known_addr { host: "127.1.19.124" port: 44787 } }
01:41:21.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:21.672325 1634 leader_election.cc:288] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 85a011fbef4b438c84f15e8e42b174c8 (127.1.19.126:40005), fce2846bed844761afbf9124220f6e42 (127.1.19.125:39053)
01:41:21.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:21.678352 1555 leader_election.cc:334] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 85a011fbef4b438c84f15e8e42b174c8 (127.1.19.126:40005): Network error: Client connection negotiation failed: client connection to 127.1.19.126:40005: connect: Connection refused (error 111)
01:41:21.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:21.678807 1553 leader_election.cc:334] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer fce2846bed844761afbf9124220f6e42 (127.1.19.125:39053): Network error: Client connection negotiation failed: client connection to 127.1.19.125:39053: connect: Connection refused (error 111)
01:41:21.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:21.679168 1553 leader_election.cc:302] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a8e7a7b6c3bc4ffebab8c3c5428bfaf4; no voters: 85a011fbef4b438c84f15e8e42b174c8, fce2846bed844761afbf9124220f6e42
01:41:21.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:21.679831 1634 raft_consensus.cc:2726] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
01:41:21.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:21.909358 1621 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:41:21.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:21.909611 1621 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:41:21.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:21.930497 1534 master_runner.cc:186] Error getting master registration for 127.1.19.126:40005: Network error: Client connection negotiation failed: client connection to 127.1.19.126:40005: connect: Connection refused (error 111),
01:41:21.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:21.930945 1534 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:41:22.309 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.125:39053(127.1.19.125:39053)] connecting to peer
01:41:22.310 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.126:40005(127.1.19.126:40005)] connecting to peer
01:41:22.312 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:22.312 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.125:39053(127.1.19.125:39053)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:22.312 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.125:39053: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:22.314 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:22.314 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.126:40005(127.1.19.126:40005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:22.314 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.126:40005: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:22.316 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787; will retry
01:41:22.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:22.913733 1621 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:41:22.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:22.914085 1621 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:41:23.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:23.916982 1621 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:41:23.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:23.917227 1621 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:41:23.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:23.919198 525 debug-util.cc:398] Leaking SignalData structure 0x7b0800099860 after lost signal to thread 460
01:41:23.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:23.919826 525 debug-util.cc:398] Leaking SignalData structure 0x7b08000ac1c0 after lost signal to thread 528
01:41:24.049 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.125:39053(127.1.19.125:39053)] connecting to peer
01:41:24.050 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.126:40005(127.1.19.126:40005)] connecting to peer
01:41:24.051 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:24.053 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.125:39053(127.1.19.125:39053)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:24.053 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:24.053 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.125:39053: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:24.053 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.126:40005(127.1.19.126:40005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:24.053 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.126:40005: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:24.058 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787; will retry
01:41:24.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:24.622378 1641 raft_consensus.cc:482] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:41:24.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:24.622725 1641 raft_consensus.cc:504] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "85a011fbef4b438c84f15e8e42b174c8" member_type: VOTER last_known_addr { host: "127.1.19.126" port: 40005 } } peers { permanent_uuid: "fce2846bed844761afbf9124220f6e42" member_type: VOTER last_known_addr { host: "127.1.19.125" port: 39053 } } peers { permanent_uuid: "a8e7a7b6c3bc4ffebab8c3c5428bfaf4" member_type: VOTER last_known_addr { host: "127.1.19.124" port: 44787 } }
01:41:24.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:24.624199 1641 leader_election.cc:288] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 85a011fbef4b438c84f15e8e42b174c8 (127.1.19.126:40005), fce2846bed844761afbf9124220f6e42 (127.1.19.125:39053)
01:41:24.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:24.626394 1555 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.126:40005: connect: Connection refused (error 111) [suppressed 5 similar messages]
01:41:24.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:24.629987 1553 leader_election.cc:334] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer fce2846bed844761afbf9124220f6e42 (127.1.19.125:39053): Network error: Client connection negotiation failed: client connection to 127.1.19.125:39053: connect: Connection refused (error 111)
01:41:24.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:24.630492 1555 leader_election.cc:334] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 85a011fbef4b438c84f15e8e42b174c8 (127.1.19.126:40005): Network error: Client connection negotiation failed: client connection to 127.1.19.126:40005: connect: Connection refused (error 111)
01:41:24.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:24.630856 1555 leader_election.cc:302] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a8e7a7b6c3bc4ffebab8c3c5428bfaf4; no voters: 85a011fbef4b438c84f15e8e42b174c8, fce2846bed844761afbf9124220f6e42
01:41:24.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:24.631479 1641 raft_consensus.cc:2726] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
01:41:24.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:24.919840 1621 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:41:24.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:24.920137 1621 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:41:25.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:25.923570 1621 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:41:25.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:25.923907 1621 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:41:26.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 01:41:26 dist-test-slave-rw6r krb5kdc[1114](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790646086, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:41:26.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:26.574836 1548 init.cc:295] Successfully reacquired a new kerberos TGT
01:41:26.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:26.926769 1621 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:41:26.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:26.927029 1621 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:41:26.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:26.934226 1534 master_runner.cc:186] Error getting master registration for 127.1.19.126:40005: Network error: Client connection negotiation failed: client connection to 127.1.19.126:40005: connect: Connection refused (error 111),
01:41:26.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:26.934614 1534 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:41:26.949 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.125:39053(127.1.19.125:39053)] connecting to peer
01:41:26.950 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.126:40005(127.1.19.126:40005)] connecting to peer
01:41:26.951 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:26.952 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.125:39053(127.1.19.125:39053)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:26.952 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.125:39053: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:26.953 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:26.953 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.126:40005(127.1.19.126:40005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:26.953 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.126:40005: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:26.956 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787; will retry
01:41:27.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:27.929668 1621 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:41:27.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:27.929922 1621 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:41:28.470 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.125:39053(127.1.19.125:39053)] connecting to peer
01:41:28.471 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.126:40005(127.1.19.126:40005)] connecting to peer
01:41:28.473 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:28.473 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.125:39053(127.1.19.125:39053)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:28.473 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.125:39053: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:28.477 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:28.477 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.126:40005(127.1.19.126:40005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:28.477 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.126:40005: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:28.479 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787; will retry
01:41:28.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:28.932668 1621 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:41:28.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:28.932932 1621 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:41:29.269 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.125:39053(127.1.19.125:39053)] connecting to peer
01:41:29.271 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.126:40005(127.1.19.126:40005)] connecting to peer
01:41:29.271 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:29.272 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.125:39053(127.1.19.125:39053)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:29.272 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.125:39053: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:29.273 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:29.273 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.126:40005(127.1.19.126:40005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:29.273 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.126:40005: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:29.275 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787; will retry
01:41:29.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:29.935820 1621 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:41:29.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:29.936084 1621 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:41:30.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:30.938905 1621 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:41:30.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:30.939158 1621 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:41:31.449 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.125:39053(127.1.19.125:39053)] connecting to peer
01:41:31.450 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.126:40005(127.1.19.126:40005)] connecting to peer
01:41:31.451 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:31.451 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.125:39053(127.1.19.125:39053)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:31.453 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.125:39053: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:31.453 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:31.453 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.126:40005(127.1.19.126:40005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:31.453 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.126:40005: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:31.457 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787; will retry
01:41:31.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:31.645478 1600 debug-util.cc:398] Leaking SignalData structure 0x7b0800097260 after lost signal to thread 1535
01:41:31.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:31.646045 1600 debug-util.cc:398] Leaking SignalData structure 0x7b080009d800 after lost signal to thread 1603
01:41:31.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:31.843065 1653 raft_consensus.cc:482] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:41:31.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:31.843647 1653 raft_consensus.cc:504] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "85a011fbef4b438c84f15e8e42b174c8" member_type: VOTER last_known_addr { host: "127.1.19.126" port: 40005 } } peers { permanent_uuid: "fce2846bed844761afbf9124220f6e42" member_type: VOTER last_known_addr { host: "127.1.19.125" port: 39053 } } peers { permanent_uuid: "a8e7a7b6c3bc4ffebab8c3c5428bfaf4" member_type: VOTER last_known_addr { host: "127.1.19.124" port: 44787 } }
01:41:31.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:31.845628 1653 leader_election.cc:288] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 85a011fbef4b438c84f15e8e42b174c8 (127.1.19.126:40005), fce2846bed844761afbf9124220f6e42 (127.1.19.125:39053)
01:41:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:31.862239 1555 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.126:40005: connect: Connection refused (error 111) [suppressed 1 similar messages]
01:41:31.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:31.868986 1553 leader_election.cc:334] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer fce2846bed844761afbf9124220f6e42 (127.1.19.125:39053): Network error: Client connection negotiation failed: client connection to 127.1.19.125:39053: connect: Connection refused (error 111)
01:41:31.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:31.869863 1555 leader_election.cc:334] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 85a011fbef4b438c84f15e8e42b174c8 (127.1.19.126:40005): Network error: Client connection negotiation failed: client connection to 127.1.19.126:40005: connect: Connection refused (error 111)
01:41:31.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:31.870277 1555 leader_election.cc:302] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a8e7a7b6c3bc4ffebab8c3c5428bfaf4; no voters: 85a011fbef4b438c84f15e8e42b174c8, fce2846bed844761afbf9124220f6e42
01:41:31.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:31.871176 1653 raft_consensus.cc:2726] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
01:41:31.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:31.943655 1621 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:41:31.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:31.944388 1621 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:41:31.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:31.982870 1534 master_runner.cc:186] Error getting master registration for 127.1.19.126:40005: Network error: Client connection negotiation failed: client connection to 127.1.19.126:40005: connect: Connection refused (error 111),
01:41:31.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:31.983374 1534 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:41:32.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:32.948849 1621 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:41:32.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:32.949226 1621 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:41:33.669 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.125:39053(127.1.19.125:39053)] connecting to peer
01:41:33.670 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.126:40005(127.1.19.126:40005)] connecting to peer
01:41:33.671 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:33.672 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.125:39053(127.1.19.125:39053)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:33.672 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.125:39053: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:33.673 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:33.673 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.126:40005(127.1.19.126:40005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:33.673 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.126:40005: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:33.676 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787; will retry
01:41:33.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:33.952207 1621 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:41:33.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:33.952463 1621 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:41:34.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:34.955267 1621 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:41:34.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:34.955557 1621 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:41:35.469 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.125:39053(127.1.19.125:39053)] connecting to peer
01:41:35.470 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.126:40005(127.1.19.126:40005)] connecting to peer
01:41:35.472 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:35.472 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.125:39053(127.1.19.125:39053)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:35.472 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.125:39053: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:35.473 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:35.473 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.126:40005(127.1.19.126:40005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:35.473 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.126:40005: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:35.476 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787; will retry
01:41:35.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:35.958791 1621 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:41:35.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:35.959138 1621 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:41:36.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 01:41:36 dist-test-slave-rw6r krb5kdc[1114](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790646096, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:41:36.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:36.577069 1548 init.cc:295] Successfully reacquired a new kerberos TGT
01:41:36.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:36.963492 1621 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:41:36.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:36.963913 1621 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:41:37.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:37.030192 1534 master_runner.cc:186] Error getting master registration for 127.1.19.126:40005: Network error: Client connection negotiation failed: client connection to 127.1.19.126:40005: connect: Connection refused (error 111),
01:41:37.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:37.030701 1534 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:41:37.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:37.226577 1664 raft_consensus.cc:482] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:41:37.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:37.227027 1664 raft_consensus.cc:504] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "85a011fbef4b438c84f15e8e42b174c8" member_type: VOTER last_known_addr { host: "127.1.19.126" port: 40005 } } peers { permanent_uuid: "fce2846bed844761afbf9124220f6e42" member_type: VOTER last_known_addr { host: "127.1.19.125" port: 39053 } } peers { permanent_uuid: "a8e7a7b6c3bc4ffebab8c3c5428bfaf4" member_type: VOTER last_known_addr { host: "127.1.19.124" port: 44787 } }
01:41:37.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:37.228754 1664 leader_election.cc:288] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 85a011fbef4b438c84f15e8e42b174c8 (127.1.19.126:40005), fce2846bed844761afbf9124220f6e42 (127.1.19.125:39053)
01:41:37.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:37.230521 1555 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.126:40005: connect: Connection refused (error 111) [suppressed 1 similar messages]
01:41:37.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:37.234159 1553 leader_election.cc:334] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer fce2846bed844761afbf9124220f6e42 (127.1.19.125:39053): Network error: Client connection negotiation failed: client connection to 127.1.19.125:39053: connect: Connection refused (error 111)
01:41:37.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:37.234763 1555 leader_election.cc:334] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 85a011fbef4b438c84f15e8e42b174c8 (127.1.19.126:40005): Network error: Client connection negotiation failed: client connection to 127.1.19.126:40005: connect: Connection refused (error 111)
01:41:37.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:37.235368 1555 leader_election.cc:302] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a8e7a7b6c3bc4ffebab8c3c5428bfaf4; no voters: 85a011fbef4b438c84f15e8e42b174c8, fce2846bed844761afbf9124220f6e42
01:41:37.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:37.235953 1664 raft_consensus.cc:2726] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
01:41:37.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:37.973003 1621 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:41:37.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:37.973269 1621 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:41:38.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:38.049218 450 debug-util.cc:398] Leaking SignalData structure 0x7b08000bb540 after lost signal to thread 387
01:41:38.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:38.049978 450 debug-util.cc:398] Leaking SignalData structure 0x7b08000a9760 after lost signal to thread 453
01:41:38.910 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.125:39053(127.1.19.125:39053)] connecting to peer
01:41:38.911 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.126:40005(127.1.19.126:40005)] connecting to peer
01:41:38.911 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:38.911 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.125:39053(127.1.19.125:39053)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:38.912 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.125:39053: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:38.913 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:38.914 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.126:40005(127.1.19.126:40005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:38.914 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.126:40005: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:38.916 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787; will retry
01:41:38.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:38.976212 1621 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:41:38.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:38.976476 1621 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:41:39.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:39.979089 1621 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:41:39.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:39.979333 1621 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:41:40.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:40.982118 1621 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:41:40.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:40.982369 1621 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:41:41.209 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.125:39053(127.1.19.125:39053)] connecting to peer
01:41:41.210 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.126:40005(127.1.19.126:40005)] connecting to peer
01:41:41.210 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:41.210 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.125:39053(127.1.19.125:39053)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:41.210 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.125:39053: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:41.211 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:41.211 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.126:40005(127.1.19.126:40005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:41.211 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.126:40005: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:41.214 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787; will retry
01:41:41.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:41.985400 1621 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:41:41.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:41.985697 1621 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:41:42.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:42.033475 1534 master_runner.cc:186] Error getting master registration for 127.1.19.126:40005: Network error: Client connection negotiation failed: client connection to 127.1.19.126:40005: connect: Connection refused (error 111),
01:41:42.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:42.033895 1534 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:41:42.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:42.809132 1673 raft_consensus.cc:482] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:41:42.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:42.809588 1673 raft_consensus.cc:504] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "85a011fbef4b438c84f15e8e42b174c8" member_type: VOTER last_known_addr { host: "127.1.19.126" port: 40005 } } peers { permanent_uuid: "fce2846bed844761afbf9124220f6e42" member_type: VOTER last_known_addr { host: "127.1.19.125" port: 39053 } } peers { permanent_uuid: "a8e7a7b6c3bc4ffebab8c3c5428bfaf4" member_type: VOTER last_known_addr { host: "127.1.19.124" port: 44787 } }
01:41:42.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:42.811245 1673 leader_election.cc:288] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 85a011fbef4b438c84f15e8e42b174c8 (127.1.19.126:40005), fce2846bed844761afbf9124220f6e42 (127.1.19.125:39053)
01:41:42.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:42.813072 1555 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.126:40005: connect: Connection refused (error 111) [suppressed 1 similar messages]
01:41:42.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:42.815665 1553 leader_election.cc:334] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer fce2846bed844761afbf9124220f6e42 (127.1.19.125:39053): Network error: Client connection negotiation failed: client connection to 127.1.19.125:39053: connect: Connection refused (error 111)
01:41:42.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:42.816058 1555 leader_election.cc:334] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 85a011fbef4b438c84f15e8e42b174c8 (127.1.19.126:40005): Network error: Client connection negotiation failed: client connection to 127.1.19.126:40005: connect: Connection refused (error 111)
01:41:42.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:42.816329 1555 leader_election.cc:302] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a8e7a7b6c3bc4ffebab8c3c5428bfaf4; no voters: 85a011fbef4b438c84f15e8e42b174c8, fce2846bed844761afbf9124220f6e42
01:41:42.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:42.816851 1673 raft_consensus.cc:2726] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
01:41:42.949 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.125:39053(127.1.19.125:39053)] connecting to peer
01:41:42.950 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.126:40005(127.1.19.126:40005)] connecting to peer
01:41:42.950 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:42.951 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.125:39053(127.1.19.125:39053)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:42.951 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.125:39053: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:42.952 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:42.952 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.126:40005(127.1.19.126:40005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:42.952 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.126:40005: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:42.954 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787; will retry
01:41:42.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:42.988447 1621 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:41:42.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:42.988787 1621 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:41:43.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:43.991510 1621 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:41:43.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:43.991834 1621 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:41:44.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:44.994441 1621 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a8e7a7b6c3bc4ffebab8c3c5428bfaf4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:41:44.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:41:44.994716 1621 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:41:45.789 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.125:39053(127.1.19.125:39053)] connecting to peer
01:41:45.790 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.126:40005(127.1.19.126:40005)] connecting to peer
01:41:45.791 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:45.791 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.125:39053(127.1.19.125:39053)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:45.791 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.125:39053: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053
01:41:45.792 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:45.792 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.126:40005(127.1.19.126:40005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:45.792 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.126:40005: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
01:41:45.795 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787; will retry
01:41:45.799 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=29068), Traces: [0ms] refreshing cache from master, [32ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [95ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [155ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [154ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [545ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [550ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [557ms] refreshing cache from master, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [559ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [561ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [562ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [564ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [568ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [569ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [577ms] refreshing cache from master, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [580ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [584ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [585ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [586ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [597ms] refreshing cache from master, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [601ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [604ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [605ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [607ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [617ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [620ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [620ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [625ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [626ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [627ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [637ms] refreshing cache from master, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [641ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [645ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [647ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [648ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [716ms] refreshing cache from master, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [720ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [720ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [724ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [726ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [727ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [757ms] refreshing cache from master, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [759ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [760ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [761ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [763ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [765ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [766ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [956ms] refreshing cache from master, [957ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [958ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [959ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [959ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [964ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [966ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [967ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [1057ms] refreshing cache from master, [1057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [1058ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [1059ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [1061ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [1063ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [1065ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [1066ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [1456ms] refreshing cache from master, [1457ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [1458ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [1459ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [1461ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [1462ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [1465ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [1466ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [2136ms] refreshing cache from master, [2136ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [2138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [2139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [2140ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [2140ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [2144ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [2144ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [5577ms] refreshing cache from master, [5577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [5578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [5579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [5580ms] too many traces: truncated at 100 traces, deferred=Deferred@946387636(state=PENDING, result=null, callback=(continuation of Deferred@995851181 after retry RPC after error@1323589291) -> (continuation of Deferred@739257152 after retry RPC after error@1372625260) -> (continuation of Deferred@825628392 after retry RPC after error@1573590883) -> (continuation of Deferred@1890352561 after retry RPC after error@635605297) -> (continuation of Deferred@610419251 after retry RPC after error@555170259) -> (continuation of Deferred@843418037 after retry RPC after error@52428245) -> (continuation of Deferred@895112287 after retry RPC after error@756755078) -> (continuation of Deferred@856495793 after retry RPC after error@1325442128) -> (continuation of Deferred@673184067 after retry RPC after error@1545910748) -> (continuation of Deferred@122127479 after retry RPC after error@1294959339) -> (continuation of Deferred@447594476 after retry RPC after error@1673400124) -> (continuation of Deferred@1491159808 after retry RPC after error@1776857645) -> (continuation of Deferred@516760099 after retry RPC after error@219748649) -> (continuation of Deferred@1601370817 after retry RPC after error@1940345560) -> (continuation of Deferred@76327605 after retry RPC after error@1092168540) -> (continuation of Deferred@158692684 after retry RPC after error@1083033384) -> (continuation of Deferred@557126798 after retry RPC after error@704550975) -> (continuation of Deferred@184764249 after retry RPC after error@630810006) -> (continuation of Deferred@746685647 after retry RPC after error@1910758912) -> (continuation of Deferred@1615554282 after retry RPC after error@244325024) -> (continuation of Deferred@773279809 after retry RPC after error@1097566301) -> (continuation of Deferred@2058148524 after retry RPC after error@1961331267) -> (continuation of Deferred@1617316294 after retry RPC after error@2110917879), errback=(continuation of Deferred@995851181 after retry RPC after error@1323589291) -> (continuation of Deferred@739257152 after retry RPC after error@1372625260) -> (continuation of Deferred@825628392 after retry RPC after error@1573590883) -> (continuation of Deferred@1890352561 after retry RPC after error@635605297) -> (continuation of Deferred@610419251 after retry RPC after error@555170259) -> (continuation of Deferred@843418037 after retry RPC after error@52428245) -> (continuation of Deferred@895112287 after retry RPC after error@756755078) -> (continuation of Deferred@856495793 after retry RPC after error@1325442128) -> (continuation of Deferred@673184067 after retry RPC after error@1545910748) -> (continuation of Deferred@122127479 after retry RPC after error@1294959339) -> (continuation of Deferred@447594476 after retry RPC after error@1673400124) -> (continuation of Deferred@1491159808 after retry RPC after error@1776857645) -> (continuation of Deferred@516760099 after retry RPC after error@219748649) -> (continuation of Deferred@1601370817 after retry RPC after error@1940345560) -> (continuation of Deferred@76327605 after retry RPC after error@1092168540) -> (continuation of Deferred@158692684 after retry RPC after error@1083033384) -> (continuation of Deferred@557126798 after retry RPC after error@704550975) -> (continuation of Deferred@184764249 after retry RPC after error@630810006) -> (continuation of Deferred@746685647 after retry RPC after error@1910758912) -> (continuation of Deferred@1615554282 after retry RPC after error@244325024) -> (continuation of Deferred@773279809 after retry RPC after error@1097566301) -> (continuation of Deferred@2058148524 after retry RPC after error@1961331267) -> (continuation of Deferred@1617316294 after retry RPC after error@2110917879)))
01:41:45.803 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.19.126:40005(127.1.19.126:40005)] disconnecting while in state TERMINATED
01:41:45.805 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.19.125:39053(127.1.19.125:39053)] disconnecting while in state TERMINATED
01:41:45.805 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.19.124:44787(127.1.19.124:44787)] disconnecting while in state READY
01:41:45.809 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
01:41:45.811 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.1.19.124:44787(127.1.19.124:44787)] handling channelInactive
01:41:45.811 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.124:44787(127.1.19.124:44787)] cleaning up while in state READY due to: connection closed
01:41:45.812 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
01:41:45.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:45.817104 1101 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task5IFP5y/build/tsan/bin/kudu with pid 1534
01:41:46.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T01:41:46Z chronyd exiting
01:41:48.035 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
01:41:48.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:48.036592 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task5IFP5y/build/tsan/bin/kudu with pid 655
01:41:48.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:48.065270 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task5IFP5y/build/tsan/bin/kudu with pid 813
01:41:48.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:48.090803 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task5IFP5y/build/tsan/bin/kudu with pid 957
01:41:48.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:48.115789 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task5IFP5y/build/tsan/bin/kudu with pid 386
01:41:48.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:48.147119 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task5IFP5y/build/tsan/bin/kudu with pid 456
01:41:48.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:41:48.177337 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task5IFP5y/build/tsan/bin/kudu with pid 531
01:41:49.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T01:41:49Z chronyd exiting
01:41:50.364 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=29068), Traces: [0ms] refreshing cache from master, [32ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [95ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [155ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [154ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [545ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [550ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [557ms] refreshing cache from master, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [559ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [561ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [562ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [564ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [568ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [569ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [577ms] refreshing cache from master, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [580ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [584ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [585ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [586ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [597ms] refreshing cache from master, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [601ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [604ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [605ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [607ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [617ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [620ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [620ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [625ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [626ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [627ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [637ms] refreshing cache from master, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [641ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [645ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [647ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [648ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [716ms] refreshing cache from master, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [720ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [720ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [724ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [726ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [727ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [757ms] refreshing cache from master, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [759ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [760ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [761ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [763ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [765ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [766ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [956ms] refreshing cache from master, [957ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [958ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [959ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [959ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [964ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [966ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [967ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [1057ms] refreshing cache from master, [1057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [1058ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [1059ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [1061ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [1063ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [1065ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [1066ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [1456ms] refreshing cache from master, [1457ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [1458ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [1459ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [1461ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [1462ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [1465ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [1466ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [2136ms] refreshing cache from master, [2136ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [2138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [2139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [2140ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [2140ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [2144ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [2144ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [5577ms] refreshing cache from master, [5577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [5578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [5579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [5580ms] too many traces: truncated at 100 traces, deferred=Deferred@946387636(state=PENDING, result=null, callback=(continuation of Deferred@995851181 after retry RPC after error@1323589291) -> (continuation of Deferred@739257152 after retry RPC after error@1372625260) -> (continuation of Deferred@825628392 after retry RPC after error@1573590883) -> (continuation of Deferred@1890352561 after retry RPC after error@635605297) -> (continuation of Deferred@610419251 after retry RPC after error@555170259) -> (continuation of Deferred@843418037 after retry RPC after error@52428245) -> (continuation of Deferred@895112287 after retry RPC after error@756755078) -> (continuation of Deferred@856495793 after retry RPC after error@1325442128) -> (continuation of Deferred@673184067 after retry RPC after error@1545910748) -> (continuation of Deferred@122127479 after retry RPC after error@1294959339) -> (continuation of Deferred@447594476 after retry RPC after error@1673400124) -> (continuation of Deferred@1491159808 after retry RPC after error@1776857645) -> (continuation of Deferred@516760099 after retry RPC after error@219748649) -> (continuation of Deferred@1601370817 after retry RPC after error@1940345560) -> (continuation of Deferred@76327605 after retry RPC after error@1092168540) -> (continuation of Deferred@158692684 after retry RPC after error@1083033384) -> (continuation of Deferred@557126798 after retry RPC after error@704550975) -> (continuation of Deferred@184764249 after retry RPC after error@630810006) -> (continuation of Deferred@746685647 after retry RPC after error@1910758912) -> (continuation of Deferred@1615554282 after retry RPC after error@244325024) -> (continuation of Deferred@773279809 after retry RPC after error@1097566301) -> (continuation of Deferred@2058148524 after retry RPC after error@1961331267) -> (continuation of Deferred@1617316294 after retry RPC after error@2110917879), errback=(continuation of Deferred@995851181 after retry RPC after error@1323589291) -> (continuation of Deferred@739257152 after retry RPC after error@1372625260) -> (continuation of Deferred@825628392 after retry RPC after error@1573590883) -> (continuation of Deferred@1890352561 after retry RPC after error@635605297) -> (continuation of Deferred@610419251 after retry RPC after error@555170259) -> (continuation of Deferred@843418037 after retry RPC after error@52428245) -> (continuation of Deferred@895112287 after retry RPC after error@756755078) -> (continuation of Deferred@856495793 after retry RPC after error@1325442128) -> (continuation of Deferred@673184067 after retry RPC after error@1545910748) -> (continuation of Deferred@122127479 after retry RPC after error@1294959339) -> (continuation of Deferred@447594476 after retry RPC after error@1673400124) -> (continuation of Deferred@1491159808 after retry RPC after error@1776857645) -> (continuation of Deferred@516760099 after retry RPC after error@219748649) -> (continuation of Deferred@1601370817 after retry RPC after error@1940345560) -> (continuation of Deferred@76327605 after retry RPC after error@1092168540) -> (continuation of Deferred@158692684 after retry RPC after error@1083033384) -> (continuation of Deferred@557126798 after retry RPC after error@704550975) -> (continuation of Deferred@184764249 after retry RPC after error@630810006) -> (continuation of Deferred@746685647 after retry RPC after error@1910758912) -> (continuation of Deferred@1615554282 after retry RPC after error@244325024) -> (continuation of Deferred@773279809 after retry RPC after error@1097566301) -> (continuation of Deferred@2058148524 after retry RPC after error@1961331267) -> (continuation of Deferred@1617316294 after retry RPC after error@2110917879)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.19.126:40005
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=29068), Traces: [0ms] refreshing cache from master, [32ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [95ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [155ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [154ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [545ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [550ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [557ms] refreshing cache from master, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [559ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [561ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [562ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [564ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [568ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [569ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [577ms] refreshing cache from master, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [580ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [584ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [585ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [586ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [597ms] refreshing cache from master, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [601ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [604ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [605ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [607ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [617ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [620ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [620ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [625ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [626ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [627ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [637ms] refreshing cache from master, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [641ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [645ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [647ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [648ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [716ms] refreshing cache from master, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [720ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [720ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [724ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [726ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [727ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [757ms] refreshing cache from master, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [759ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [760ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [761ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [763ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [765ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [766ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [956ms] refreshing cache from master, [957ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [958ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [959ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [959ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [964ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [966ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [967ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [1057ms] refreshing cache from master, [1057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [1058ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [1059ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [1061ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [1063ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [1065ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [1066ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [1456ms] refreshing cache from master, [1457ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [1458ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [1459ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [1461ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [1462ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [1465ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [1466ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [2136ms] refreshing cache from master, [2136ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [2138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [2139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [2140ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [2140ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [2144ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [2144ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [5577ms] refreshing cache from master, [5577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [5578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [5579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [5580ms] too many traces: truncated at 100 traces, deferred=Deferred@946387636(state=PENDING, result=null, callback=(continuation of Deferred@995851181 after retry RPC after error@1323589291) -> (continuation of Deferred@739257152 after retry RPC after error@1372625260) -> (continuation of Deferred@825628392 after retry RPC after error@1573590883) -> (continuation of Deferred@1890352561 after retry RPC after error@635605297) -> (continuation of Deferred@610419251 after retry RPC after error@555170259) -> (continuation of Deferred@843418037 after retry RPC after error@52428245) -> (continuation of Deferred@895112287 after retry RPC after error@756755078) -> (continuation of Deferred@856495793 after retry RPC after error@1325442128) -> (continuation of Deferred@673184067 after retry RPC after error@1545910748) -> (continuation of Deferred@122127479 after retry RPC after error@1294959339) -> (continuation of Deferred@447594476 after retry RPC after error@1673400124) -> (continuation of Deferred@1491159808 after retry RPC after error@1776857645) -> (continuation of Deferred@516760099 after retry RPC after error@219748649) -> (continuation of Deferred@1601370817 after retry RPC after error@1940345560) -> (continuation of Deferred@76327605 after retry RPC after error@1092168540) -> (continuation of Deferred@158692684 after retry RPC after error@1083033384) -> (continuation of Deferred@557126798 after retry RPC after error@704550975) -> (continuation of Deferred@184764249 after retry RPC after error@630810006) -> (continuation of Deferred@746685647 after retry RPC after error@1910758912) -> (continuation of Deferred@1615554282 after retry RPC after error@244325024) -> (continuation of Deferred@773279809 after retry RPC after error@1097566301) -> (continuation of Deferred@2058148524 after retry RPC after error@1961331267) -> (continuation of Deferred@1617316294 after retry RPC after error@2110917879), errback=(continuation of Deferred@995851181 after retry RPC after error@1323589291) -> (continuation of Deferred@739257152 after retry RPC after error@1372625260) -> (continuation of Deferred@825628392 after retry RPC after error@1573590883) -> (continuation of Deferred@1890352561 after retry RPC after error@635605297) -> (continuation of Deferred@610419251 after retry RPC after error@555170259) -> (continuation of Deferred@843418037 after retry RPC after error@52428245) -> (continuation of Deferred@895112287 after retry RPC after error@756755078) -> (continuation of Deferred@856495793 after retry RPC after error@1325442128) -> (continuation of Deferred@673184067 after retry RPC after error@1545910748) -> (continuation of Deferred@122127479 after retry RPC after error@1294959339) -> (continuation of Deferred@447594476 after retry RPC after error@1673400124) -> (continuation of Deferred@1491159808 after retry RPC after error@1776857645) -> (continuation of Deferred@516760099 after retry RPC after error@219748649) -> (continuation of Deferred@1601370817 after retry RPC after error@1940345560) -> (continuation of Deferred@76327605 after retry RPC after error@1092168540) -> (continuation of Deferred@158692684 after retry RPC after error@1083033384) -> (continuation of Deferred@557126798 after retry RPC after error@704550975) -> (continuation of Deferred@184764249 after retry RPC after error@630810006) -> (continuation of Deferred@746685647 after retry RPC after error@1910758912) -> (continuation of Deferred@1615554282 after retry RPC after error@244325024) -> (continuation of Deferred@773279809 after retry RPC after error@1097566301) -> (continuation of Deferred@2058148524 after retry RPC after error@1961331267) -> (continuation of Deferred@1617316294 after retry RPC after error@2110917879)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.19.126:40005
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
01:41:51.068 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
01:41:51.068 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=29068), Traces: [0ms] refreshing cache from master, [32ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [95ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [155ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [154ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [545ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [550ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [557ms] refreshing cache from master, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [559ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [561ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [562ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [564ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [568ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [569ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [577ms] refreshing cache from master, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [580ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [584ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [585ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [586ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [597ms] refreshing cache from master, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [601ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [604ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [605ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [607ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [617ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [620ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [620ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [625ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [626ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [627ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [637ms] refreshing cache from master, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [641ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [645ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [647ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [648ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [716ms] refreshing cache from master, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [720ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [720ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [724ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [726ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [727ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [757ms] refreshing cache from master, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [759ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [760ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [761ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [763ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [765ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [766ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [956ms] refreshing cache from master, [957ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [958ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [959ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [959ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [964ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [966ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [967ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [1057ms] refreshing cache from master, [1057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [1058ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [1059ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [1061ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [1063ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [1065ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [1066ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [1456ms] refreshing cache from master, [1457ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [1458ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [1459ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [1461ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [1462ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [1465ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [1466ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [2136ms] refreshing cache from master, [2136ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [2138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [2139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [2140ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [2140ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [2144ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [2144ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [5577ms] refreshing cache from master, [5577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [5578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [5579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [5580ms] too many traces: truncated at 100 traces, deferred=Deferred@946387636(state=PENDING, result=null, callback=(continuation of Deferred@995851181 after retry RPC after error@1323589291) -> (continuation of Deferred@739257152 after retry RPC after error@1372625260) -> (continuation of Deferred@825628392 after retry RPC after error@1573590883) -> (continuation of Deferred@1890352561 after retry RPC after error@635605297) -> (continuation of Deferred@610419251 after retry RPC after error@555170259) -> (continuation of Deferred@843418037 after retry RPC after error@52428245) -> (continuation of Deferred@895112287 after retry RPC after error@756755078) -> (continuation of Deferred@856495793 after retry RPC after error@1325442128) -> (continuation of Deferred@673184067 after retry RPC after error@1545910748) -> (continuation of Deferred@122127479 after retry RPC after error@1294959339) -> (continuation of Deferred@447594476 after retry RPC after error@1673400124) -> (continuation of Deferred@1491159808 after retry RPC after error@1776857645) -> (continuation of Deferred@516760099 after retry RPC after error@219748649) -> (continuation of Deferred@1601370817 after retry RPC after error@1940345560) -> (continuation of Deferred@76327605 after retry RPC after error@1092168540) -> (continuation of Deferred@158692684 after retry RPC after error@1083033384) -> (continuation of Deferred@557126798 after retry RPC after error@704550975) -> (continuation of Deferred@184764249 after retry RPC after error@630810006) -> (continuation of Deferred@746685647 after retry RPC after error@1910758912) -> (continuation of Deferred@1615554282 after retry RPC after error@244325024) -> (continuation of Deferred@773279809 after retry RPC after error@1097566301) -> (continuation of Deferred@2058148524 after retry RPC after error@1961331267) -> (continuation of Deferred@1617316294 after retry RPC after error@2110917879), errback=(continuation of Deferred@995851181 after retry RPC after error@1323589291) -> (continuation of Deferred@739257152 after retry RPC after error@1372625260) -> (continuation of Deferred@825628392 after retry RPC after error@1573590883) -> (continuation of Deferred@1890352561 after retry RPC after error@635605297) -> (continuation of Deferred@610419251 after retry RPC after error@555170259) -> (continuation of Deferred@843418037 after retry RPC after error@52428245) -> (continuation of Deferred@895112287 after retry RPC after error@756755078) -> (continuation of Deferred@856495793 after retry RPC after error@1325442128) -> (continuation of Deferred@673184067 after retry RPC after error@1545910748) -> (continuation of Deferred@122127479 after retry RPC after error@1294959339) -> (continuation of Deferred@447594476 after retry RPC after error@1673400124) -> (continuation of Deferred@1491159808 after retry RPC after error@1776857645) -> (continuation of Deferred@516760099 after retry RPC after error@219748649) -> (continuation of Deferred@1601370817 after retry RPC after error@1940345560) -> (continuation of Deferred@76327605 after retry RPC after error@1092168540) -> (continuation of Deferred@158692684 after retry RPC after error@1083033384) -> (continuation of Deferred@557126798 after retry RPC after error@704550975) -> (continuation of Deferred@184764249 after retry RPC after error@630810006) -> (continuation of Deferred@746685647 after retry RPC after error@1910758912) -> (continuation of Deferred@1615554282 after retry RPC after error@244325024) -> (continuation of Deferred@773279809 after retry RPC after error@1097566301) -> (continuation of Deferred@2058148524 after retry RPC after error@1961331267) -> (continuation of Deferred@1617316294 after retry RPC after error@2110917879)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.19.126:40005
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=29068), Traces: [0ms] refreshing cache from master, [32ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [95ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [155ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [154ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [545ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [550ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [557ms] refreshing cache from master, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [559ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [561ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [562ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [564ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [568ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [569ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [577ms] refreshing cache from master, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [580ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [584ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [585ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [586ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [597ms] refreshing cache from master, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [601ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [604ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [605ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [607ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [617ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [620ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [620ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [625ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [626ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [627ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [637ms] refreshing cache from master, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [641ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [645ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [647ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [648ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [716ms] refreshing cache from master, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [720ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [720ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [724ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [726ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [727ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [757ms] refreshing cache from master, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [759ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [760ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [761ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [763ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [765ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [766ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [956ms] refreshing cache from master, [957ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [958ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [959ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [959ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [964ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [966ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [967ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [1057ms] refreshing cache from master, [1057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [1058ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [1059ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [1061ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [1063ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [1065ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [1066ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [1456ms] refreshing cache from master, [1457ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [1458ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [1459ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [1461ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [1462ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [1465ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [1466ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [2136ms] refreshing cache from master, [2136ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [2138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [2139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [2140ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.125:39053: Network error: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053, [2140ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.126:40005: Network error: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [2144ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.124:44787: OK, [2144ms] delaying RPC due to: Service unavailable: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005, [5577ms] refreshing cache from master, [5577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.125:39053, [5578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.126:40005, [5579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.124:44787, [5580ms] too many traces: truncated at 100 traces, deferred=Deferred@946387636(state=PENDING, result=null, callback=(continuation of Deferred@995851181 after retry RPC after error@1323589291) -> (continuation of Deferred@739257152 after retry RPC after error@1372625260) -> (continuation of Deferred@825628392 after retry RPC after error@1573590883) -> (continuation of Deferred@1890352561 after retry RPC after error@635605297) -> (continuation of Deferred@610419251 after retry RPC after error@555170259) -> (continuation of Deferred@843418037 after retry RPC after error@52428245) -> (continuation of Deferred@895112287 after retry RPC after error@756755078) -> (continuation of Deferred@856495793 after retry RPC after error@1325442128) -> (continuation of Deferred@673184067 after retry RPC after error@1545910748) -> (continuation of Deferred@122127479 after retry RPC after error@1294959339) -> (continuation of Deferred@447594476 after retry RPC after error@1673400124) -> (continuation of Deferred@1491159808 after retry RPC after error@1776857645) -> (continuation of Deferred@516760099 after retry RPC after error@219748649) -> (continuation of Deferred@1601370817 after retry RPC after error@1940345560) -> (continuation of Deferred@76327605 after retry RPC after error@1092168540) -> (continuation of Deferred@158692684 after retry RPC after error@1083033384) -> (continuation of Deferred@557126798 after retry RPC after error@704550975) -> (continuation of Deferred@184764249 after retry RPC after error@630810006) -> (continuation of Deferred@746685647 after retry RPC after error@1910758912) -> (continuation of Deferred@1615554282 after retry RPC after error@244325024) -> (continuation of Deferred@773279809 after retry RPC after error@1097566301) -> (continuation of Deferred@2058148524 after retry RPC after error@1961331267) -> (continuation of Deferred@1617316294 after retry RPC after error@2110917879), errback=(continuation of Deferred@995851181 after retry RPC after error@1323589291) -> (continuation of Deferred@739257152 after retry RPC after error@1372625260) -> (continuation of Deferred@825628392 after retry RPC after error@1573590883) -> (continuation of Deferred@1890352561 after retry RPC after error@635605297) -> (continuation of Deferred@610419251 after retry RPC after error@555170259) -> (continuation of Deferred@843418037 after retry RPC after error@52428245) -> (continuation of Deferred@895112287 after retry RPC after error@756755078) -> (continuation of Deferred@856495793 after retry RPC after error@1325442128) -> (continuation of Deferred@673184067 after retry RPC after error@1545910748) -> (continuation of Deferred@122127479 after retry RPC after error@1294959339) -> (continuation of Deferred@447594476 after retry RPC after error@1673400124) -> (continuation of Deferred@1491159808 after retry RPC after error@1776857645) -> (continuation of Deferred@516760099 after retry RPC after error@219748649) -> (continuation of Deferred@1601370817 after retry RPC after error@1940345560) -> (continuation of Deferred@76327605 after retry RPC after error@1092168540) -> (continuation of Deferred@158692684 after retry RPC after error@1083033384) -> (continuation of Deferred@557126798 after retry RPC after error@704550975) -> (continuation of Deferred@184764249 after retry RPC after error@630810006) -> (continuation of Deferred@746685647 after retry RPC after error@1910758912) -> (continuation of Deferred@1615554282 after retry RPC after error@244325024) -> (continuation of Deferred@773279809 after retry RPC after error@1097566301) -> (continuation of Deferred@2058148524 after retry RPC after error@1961331267) -> (continuation of Deferred@1617316294 after retry RPC after error@2110917879)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.19.125:39053,127.1.19.126:40005,127.1.19.124:44787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.125:39053(127.1.19.125:39053): Connection refused: /127.1.19.125:39053,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.126:40005(127.1.19.126:40005): Connection refused: /127.1.19.126:40005
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.19.126:40005
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)