Unable to diagnose

21:52:28.333 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
21:52:28.333 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-tasktmwPRy/test-tmp
21:52:28.333 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
21:52:28.333 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-tasktmwPRy/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
21:52:28.335 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
  num_masters: 3
  num_tservers: 3
  enable_kerberos: false
  cluster_root: "/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522"
  hms_mode: NONE
  mini_kdc_options {
  }
  principal: "kudu"
  mini_oidc_options {
  }
}

21:52:28.689 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
21:52:28.689 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}

21:52:28.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:28.692863  9331 mini_oidc.cc:150] Starting JWKS server
21:52:28.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:28.697180  9331 webserver.cc:480] Webserver started at http://127.0.0.1:43799/ using document root <none> and password file <none>
21:52:28.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:28.697682  9331 mini_oidc.cc:186] Starting OIDC Discovery server
21:52:28.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:28.699957  9331 webserver.cc:480] Webserver started at http://127.0.0.1:39673/ using document root <none> and password file <none>
21:52:28.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-02T21:52:28Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
21:52:28.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-02T21:52:28Z Disabled control of system clock
21:52:28.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:28.773836  9331 external_mini_cluster.cc:1366] Running /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu
21:52:28.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu
21:52:28.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/master-0/wal
21:52:28.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/master-0/data
21:52:28.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:52:28.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:52:28.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:52:28.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:52:28.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:52:28.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:52:28.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/master-0/logs
21:52:28.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/master-0/data/info.pb
21:52:28.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:52:28.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:52:28.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:52:28.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:52:28.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:52:28.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:52:28.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:52:28.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:52:28.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:52:28.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:52:28.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:52:28.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:52:28.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.28.254:37687
21:52:28.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.28.254
21:52:28.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:52:28.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.28.212:34313
21:52:28.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:52:28.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:52:28.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:52:28.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:52:28.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.28.254:37687,127.9.28.253:41061,127.9.28.252:40867
21:52:28.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:52:28.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43799/jwks.json with env {}
21:52:29.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:29.112483  9343 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:52:29.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:29.113083  9343 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:52:29.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:29.113581  9343 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:52:29.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:29.149307  9343 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:52:29.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:29.149634  9343 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:52:29.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:29.149837  9343 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:52:29.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:29.150054  9343 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:52:29.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:29.150254  9343 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:52:29.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:29.150542  9343 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43799/jwks.json
21:52:29.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:29.189833  9343 master_runner.cc:386] Master server non-default flags:
21:52:29.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:52:29.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.28.212:34313
21:52:29.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:52:29.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:52:29.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/master-0/data
21:52:29.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/master-0/wal
21:52:29.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:52:29.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.28.254:37687,127.9.28.253:41061,127.9.28.252:40867
21:52:29.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:52:29.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:52:29.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:52:29.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.28.254:37687
21:52:29.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:52:29.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:52:29.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:52:29.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43799/jwks.json
21:52:29.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:52:29.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:52:29.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/master-0/data/info.pb
21:52:29.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.28.254
21:52:29.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:52:29.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:52:29.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:52:29.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:52:29.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:52:29.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:52:29.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/master-0/logs
21:52:29.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:52:29.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:52:29.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:52:29.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:52:29.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:52:29.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6ab3f249351a4ee5d73eb8bda3585c4dcd85d070
21:52:29.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:52:29.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 21:30:03 UTC on 5fd53c4cbb9d
21:52:29.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7869
21:52:29.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:52:29.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:29.191223  9343 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:52:29.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:29.193006  9343 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:52:29.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:29.205552  9349 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:52:30.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:29.206285  9350 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:52:30.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:30.544765  9352 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:52:30.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:30.549582  9343 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.344s	user 0.410s	sys 0.930s
21:52:30.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:30.549852  9351 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1342 milliseconds
21:52:30.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:30.550011  9343 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.345s	user 0.410s	sys 0.930s
21:52:30.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:30.550346  9343 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:52:30.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:30.552112  9343 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:52:30.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:30.555862  9343 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:52:30.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:30.557644  9343 hybrid_clock.cc:648] HybridClock initialized: now 1756849950557583 us; error 49 us; skew 500 ppm
21:52:30.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:30.559454  9343 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:52:30.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:30.569977  9343 webserver.cc:480] Webserver started at http://127.9.28.254:42375/ using document root <none> and password file <none>
21:52:30.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:30.571944  9343 fs_manager.cc:362] Metadata directory not provided
21:52:30.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:30.572360  9343 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:52:30.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:30.573053  9343 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:52:30.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:30.581266  9343 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/master-0/data/instance:
21:52:30.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d05473426f1e4e84a5fc4af172cc93f9"
21:52:30.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:52:30 on dist-test-slave-q7qm"
21:52:30.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:30.584091  9343 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/master-0/wal/instance:
21:52:30.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d05473426f1e4e84a5fc4af172cc93f9"
21:52:30.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:52:30 on dist-test-slave-q7qm"
21:52:30.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:30.596428  9343 fs_manager.cc:696] Time spent creating directory manager: real 0.011s	user 0.010s	sys 0.002s
21:52:30.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:30.606523  9359 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:52:30.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:30.608650  9343 fs_manager.cc:730] Time spent opening block manager: real 0.006s	user 0.003s	sys 0.003s
21:52:30.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:30.609136  9343 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/master-0/data,/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/master-0/wal
21:52:30.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d05473426f1e4e84a5fc4af172cc93f9"
21:52:30.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:52:30 on dist-test-slave-q7qm"
21:52:30.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:30.609671  9343 fs_report.cc:389] FS layout report
21:52:30.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:52:30.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/master-0/wal
21:52:30.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/master-0/wal
21:52:30.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/master-0/data/data
21:52:30.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:52:30.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:52:30.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:52:30.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:52:30.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:52:30.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:52:30.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:52:30.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:52:30.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:52:30.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:52:30.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:52:30.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:30.718966  9343 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:52:30.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:30.720772  9343 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:52:30.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:30.721256  9343 kserver.cc:163] Server-wide thread pool size limit: 3276
21:52:30.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:30.804667  9343 rpc_server.cc:307] RPC server started. Bound to: 127.9.28.254:37687
21:52:30.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:30.804771  9410 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.28.254:37687 every 8 connection(s)
21:52:30.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:30.807708  9343 server_base.cc:1179] Dumped server information to /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/master-0/data/info.pb
21:52:30.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:30.810498  9331 external_mini_cluster.cc:1428] Started /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu as pid 9343
21:52:30.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:30.811137  9331 external_mini_cluster.cc:1442] Reading /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/master-0/wal/instance
21:52:30.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:30.813920  9411 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:52:30.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:30.814914  9331 external_mini_cluster.cc:1366] Running /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu
21:52:30.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu
21:52:30.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/master-1/wal
21:52:30.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/master-1/data
21:52:30.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:52:30.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:52:30.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:52:30.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:52:30.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:52:30.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:52:30.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/master-1/logs
21:52:30.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/master-1/data/info.pb
21:52:30.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:52:30.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:52:30.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:52:30.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:52:30.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:52:30.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:52:30.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:52:30.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:52:30.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:52:30.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:52:30.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:52:30.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:52:30.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.28.253:41061
21:52:30.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.28.253
21:52:30.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:52:30.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.28.212:34313
21:52:30.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:52:30.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:52:30.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:52:30.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:52:30.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.28.254:37687,127.9.28.253:41061,127.9.28.252:40867
21:52:30.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:52:30.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43799/jwks.json with env {}
21:52:30.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:30.826862  9411 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.28.254" port: 37687 } has no permanent_uuid. Determining permanent_uuid...
21:52:30.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:30.861258  9411 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.28.253" port: 41061 } has no permanent_uuid. Determining permanent_uuid...
21:52:30.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:30.864063  9362 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.28.253:41061: connect: Connection refused (error 111)
21:52:30.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:30.868811  9411 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.9.28.253:41061: Network error: Client connection negotiation failed: client connection to 127.9.28.253:41061: connect: Connection refused (error 111)
21:52:30.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:30.918418  9411 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.28.253" port: 41061 } attempt: 1
21:52:30.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:30.923988  9411 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.9.28.253:41061: Network error: Client connection negotiation failed: client connection to 127.9.28.253:41061: connect: Connection refused (error 111)
21:52:30.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:30.992436  9411 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.28.253" port: 41061 } attempt: 2
21:52:30.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:30.996480  9411 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.9.28.253:41061: Network error: Client connection negotiation failed: client connection to 127.9.28.253:41061: connect: Connection refused (error 111)
21:52:31.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:31.088016  9411 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.28.253" port: 41061 } attempt: 3
21:52:31.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:31.093132  9411 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.9.28.253:41061: Network error: Client connection negotiation failed: client connection to 127.9.28.253:41061: connect: Connection refused (error 111)
21:52:31.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:31.162014  9413 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:52:31.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:31.162786  9413 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:52:31.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:31.163406  9413 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:52:31.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:31.197532  9413 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:52:31.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:31.197911  9413 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:52:31.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:31.198248  9413 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:52:31.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:31.198539  9413 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:52:31.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:31.198812  9413 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:52:31.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:31.199034  9413 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43799/jwks.json
21:52:31.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:31.236699  9411 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.28.253" port: 41061 } attempt: 4
21:52:31.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:31.241479  9411 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.9.28.253:41061: Network error: Client connection negotiation failed: client connection to 127.9.28.253:41061: connect: Connection refused (error 111)
21:52:31.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:31.241515  9413 master_runner.cc:386] Master server non-default flags:
21:52:31.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:52:31.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.28.212:34313
21:52:31.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:52:31.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:52:31.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/master-1/data
21:52:31.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/master-1/wal
21:52:31.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:52:31.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.28.254:37687,127.9.28.253:41061,127.9.28.252:40867
21:52:31.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:52:31.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:52:31.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:52:31.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.28.253:41061
21:52:31.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:52:31.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:52:31.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:52:31.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43799/jwks.json
21:52:31.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:52:31.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:52:31.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/master-1/data/info.pb
21:52:31.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.28.253
21:52:31.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:52:31.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:52:31.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:52:31.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:52:31.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:52:31.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:52:31.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/master-1/logs
21:52:31.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:52:31.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:52:31.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:52:31.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:52:31.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:52:31.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6ab3f249351a4ee5d73eb8bda3585c4dcd85d070
21:52:31.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:52:31.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 21:30:03 UTC on 5fd53c4cbb9d
21:52:31.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7869
21:52:31.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:52:31.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:31.242933  9413 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:52:31.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:31.245170  9413 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:52:31.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:31.258473  9422 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:52:31.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:31.541129  9411 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.28.253" port: 41061 } attempt: 5
21:52:31.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:31.549170  9411 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.9.28.253:41061: Network error: Client connection negotiation failed: client connection to 127.9.28.253:41061: connect: Connection refused (error 111)
21:52:32.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:32.096987  9411 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.28.253" port: 41061 } attempt: 6
21:52:32.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:32.109272  9411 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.9.28.253:41061: Network error: Client connection negotiation failed: client connection to 127.9.28.253:41061: connect: Connection refused (error 111)
21:52:32.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:31.263104  9425 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:52:32.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:31.258777  9423 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:52:32.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:32.622123  9424 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1361 milliseconds
21:52:32.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:32.622267  9413 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:52:32.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:32.623997  9413 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:52:32.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:32.627362  9413 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:52:32.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:32.629045  9413 hybrid_clock.cc:648] HybridClock initialized: now 1756849952628949 us; error 85 us; skew 500 ppm
21:52:32.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:32.630409  9413 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:52:32.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:32.636929  9413 webserver.cc:480] Webserver started at http://127.9.28.253:33853/ using document root <none> and password file <none>
21:52:32.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:32.637909  9413 fs_manager.cc:362] Metadata directory not provided
21:52:32.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:32.638121  9413 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:52:32.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:32.638547  9413 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:52:32.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:32.644691  9413 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/master-1/data/instance:
21:52:32.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "537228d41a4741d39050ca442dc448c9"
21:52:32.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:52:32 on dist-test-slave-q7qm"
21:52:32.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:32.645920  9413 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/master-1/wal/instance:
21:52:32.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "537228d41a4741d39050ca442dc448c9"
21:52:32.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:52:32 on dist-test-slave-q7qm"
21:52:32.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:32.653796  9413 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.001s	sys 0.008s
21:52:32.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:32.660151  9434 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:52:32.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:32.661422  9413 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.002s	sys 0.002s
21:52:32.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:32.661839  9413 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/master-1/data,/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/master-1/wal
21:52:32.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "537228d41a4741d39050ca442dc448c9"
21:52:32.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:52:32 on dist-test-slave-q7qm"
21:52:32.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:32.662213  9413 fs_report.cc:389] FS layout report
21:52:32.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:52:32.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/master-1/wal
21:52:32.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/master-1/wal
21:52:32.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/master-1/data/data
21:52:32.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:52:32.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:52:32.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:52:32.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:52:32.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:52:32.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:52:32.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:52:32.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:52:32.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:52:32.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:52:32.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:52:32.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:32.730103  9413 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:52:32.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:32.731819  9413 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:52:32.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:32.732363  9413 kserver.cc:163] Server-wide thread pool size limit: 3276
21:52:32.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:32.821015  9413 rpc_server.cc:307] RPC server started. Bound to: 127.9.28.253:41061
21:52:32.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:32.821110  9485 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.28.253:41061 every 8 connection(s)
21:52:32.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:32.824610  9413 server_base.cc:1179] Dumped server information to /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/master-1/data/info.pb
21:52:32.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:32.824950  9331 external_mini_cluster.cc:1428] Started /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu as pid 9413
21:52:32.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:32.825381  9331 external_mini_cluster.cc:1442] Reading /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/master-1/wal/instance
21:52:32.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:32.828212  9331 external_mini_cluster.cc:1366] Running /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu
21:52:32.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu
21:52:32.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/master-2/wal
21:52:32.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/master-2/data
21:52:32.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:52:32.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:52:32.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:52:32.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:52:32.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:52:32.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:52:32.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/master-2/logs
21:52:32.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/master-2/data/info.pb
21:52:32.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:52:32.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:52:32.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:52:32.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:52:32.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:52:32.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:52:32.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:52:32.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:52:32.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:52:32.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:52:32.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:52:32.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:52:32.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.28.252:40867
21:52:32.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.28.252
21:52:32.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:52:32.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.28.212:34313
21:52:32.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:52:32.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:52:32.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:52:32.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:52:32.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.28.254:37687,127.9.28.253:41061,127.9.28.252:40867
21:52:32.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:52:32.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43799/jwks.json with env {}
21:52:32.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:32.832319  9486 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:52:32.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:32.844296  9486 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.28.254" port: 37687 } has no permanent_uuid. Determining permanent_uuid...
21:52:32.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:32.870854  9486 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.28.253" port: 41061 } has no permanent_uuid. Determining permanent_uuid...
21:52:32.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:32.889247  9486 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.28.252" port: 40867 } has no permanent_uuid. Determining permanent_uuid...
21:52:32.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:32.892124  9435 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.28.252:40867: connect: Connection refused (error 111)
21:52:32.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:32.896869  9486 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.9.28.252:40867: Network error: Client connection negotiation failed: client connection to 127.9.28.252:40867: connect: Connection refused (error 111)
21:52:32.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:32.946385  9486 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.28.252" port: 40867 } attempt: 1
21:52:32.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:32.953999  9486 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.9.28.252:40867: Network error: Client connection negotiation failed: client connection to 127.9.28.252:40867: connect: Connection refused (error 111)
21:52:33.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:33.022552  9486 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.28.252" port: 40867 } attempt: 2
21:52:33.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:33.026942  9486 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.9.28.252:40867: Network error: Client connection negotiation failed: client connection to 127.9.28.252:40867: connect: Connection refused (error 111)
21:52:33.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:33.118468  9486 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.28.252" port: 40867 } attempt: 3
21:52:33.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:33.123091  9486 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.9.28.252:40867: Network error: Client connection negotiation failed: client connection to 127.9.28.252:40867: connect: Connection refused (error 111)
21:52:33.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:33.169927  9411 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.28.253" port: 41061 } attempt: 7
21:52:33.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:33.182860  9411 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.28.252" port: 40867 } has no permanent_uuid. Determining permanent_uuid...
21:52:33.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:33.188592  9411 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.9.28.252:40867: Network error: Client connection negotiation failed: client connection to 127.9.28.252:40867: connect: Connection refused (error 111)
21:52:33.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:33.198714  9488 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:52:33.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:33.199278  9488 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:52:33.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:33.199836  9488 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:52:33.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:33.236232  9488 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:52:33.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:33.236548  9488 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:52:33.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:33.236749  9488 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:52:33.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:33.236948  9488 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:52:33.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:33.237155  9488 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:52:33.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:33.237344  9488 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43799/jwks.json
21:52:33.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:33.247115  9411 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.28.252" port: 40867 } attempt: 1
21:52:33.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:33.251292  9411 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.9.28.252:40867: Network error: Client connection negotiation failed: client connection to 127.9.28.252:40867: connect: Connection refused (error 111)
21:52:33.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:33.266649  9486 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.28.252" port: 40867 } attempt: 4
21:52:33.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:33.271783  9486 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.9.28.252:40867: Network error: Client connection negotiation failed: client connection to 127.9.28.252:40867: connect: Connection refused (error 111)
21:52:33.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:33.278959  9488 master_runner.cc:386] Master server non-default flags:
21:52:33.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:52:33.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.28.212:34313
21:52:33.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:52:33.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:52:33.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/master-2/data
21:52:33.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/master-2/wal
21:52:33.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:52:33.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.28.254:37687,127.9.28.253:41061,127.9.28.252:40867
21:52:33.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:52:33.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:52:33.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:52:33.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.28.252:40867
21:52:33.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:52:33.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:52:33.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:52:33.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43799/jwks.json
21:52:33.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:52:33.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:52:33.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/master-2/data/info.pb
21:52:33.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.28.252
21:52:33.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:52:33.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:52:33.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:52:33.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:52:33.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:52:33.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:52:33.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/master-2/logs
21:52:33.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:52:33.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:52:33.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:52:33.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:52:33.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:52:33.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6ab3f249351a4ee5d73eb8bda3585c4dcd85d070
21:52:33.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:52:33.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 21:30:03 UTC on 5fd53c4cbb9d
21:52:33.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7869
21:52:33.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:52:33.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:33.280366  9488 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:52:33.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:33.282837  9488 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:52:33.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:33.298401  9500 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:52:33.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:33.332859  9411 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.28.252" port: 40867 } attempt: 2
21:52:33.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:33.340505  9411 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.9.28.252:40867: Network error: Client connection negotiation failed: client connection to 127.9.28.252:40867: connect: Connection refused (error 111)
21:52:33.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:33.426409  9411 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.28.252" port: 40867 } attempt: 3
21:52:33.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:33.434124  9411 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.9.28.252:40867: Network error: Client connection negotiation failed: client connection to 127.9.28.252:40867: connect: Connection refused (error 111)
21:52:33.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:33.571285  9486 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.28.252" port: 40867 } attempt: 5
21:52:33.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:33.574720  9411 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.28.252" port: 40867 } attempt: 4
21:52:33.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:33.579831  9486 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.9.28.252:40867: Network error: Client connection negotiation failed: client connection to 127.9.28.252:40867: connect: Connection refused (error 111)
21:52:33.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:33.583539  9411 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.9.28.252:40867: Network error: Client connection negotiation failed: client connection to 127.9.28.252:40867: connect: Connection refused (error 111)
21:52:33.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:33.867205  9411 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.28.252" port: 40867 } attempt: 5
21:52:33.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:33.878827  9411 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.9.28.252:40867: Network error: Client connection negotiation failed: client connection to 127.9.28.252:40867: connect: Connection refused (error 111)
21:52:34.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:34.127487  9486 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.28.252" port: 40867 } attempt: 6
21:52:34.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:34.142081  9486 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.9.28.252:40867: Network error: Client connection negotiation failed: client connection to 127.9.28.252:40867: connect: Connection refused (error 111)
21:52:34.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:34.431486  9411 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.28.252" port: 40867 } attempt: 6
21:52:34.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:34.443599  9411 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.9.28.252:40867: Network error: Client connection negotiation failed: client connection to 127.9.28.252:40867: connect: Connection refused (error 111)
21:52:34.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:33.298812  9501 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:52:34.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:34.614821  9503 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:52:34.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:34.624311  9502 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1314 milliseconds
21:52:34.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:34.625303  9488 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:52:34.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:34.626665  9488 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:52:34.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:34.629848  9488 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:52:34.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:34.631290  9488 hybrid_clock.cc:648] HybridClock initialized: now 1756849954631246 us; error 64 us; skew 500 ppm
21:52:34.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:34.632175  9488 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:52:34.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:34.639850  9488 webserver.cc:480] Webserver started at http://127.9.28.252:33487/ using document root <none> and password file <none>
21:52:34.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:34.641137  9488 fs_manager.cc:362] Metadata directory not provided
21:52:34.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:34.641428  9488 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:52:34.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:34.641975  9488 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:52:34.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:34.647922  9488 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/master-2/data/instance:
21:52:34.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "574662fc86ca4395937a670f6394b8d0"
21:52:34.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:52:34 on dist-test-slave-q7qm"
21:52:34.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:34.649806  9488 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/master-2/wal/instance:
21:52:34.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "574662fc86ca4395937a670f6394b8d0"
21:52:34.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:52:34 on dist-test-slave-q7qm"
21:52:34.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:34.658172  9488 fs_manager.cc:696] Time spent creating directory manager: real 0.008s	user 0.005s	sys 0.004s
21:52:34.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:34.664577  9514 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:52:34.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:34.666010  9488 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.002s
21:52:34.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:34.666369  9488 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/master-2/data,/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/master-2/wal
21:52:34.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "574662fc86ca4395937a670f6394b8d0"
21:52:34.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:52:34 on dist-test-slave-q7qm"
21:52:34.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:34.666712  9488 fs_report.cc:389] FS layout report
21:52:34.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:52:34.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/master-2/wal
21:52:34.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/master-2/wal
21:52:34.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/master-2/data/data
21:52:34.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:52:34.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:52:34.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:52:34.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:52:34.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:52:34.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:52:34.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:52:34.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:52:34.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:52:34.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:52:34.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:52:34.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:34.726931  9488 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:52:34.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:34.729121  9488 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:52:34.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:34.729622  9488 kserver.cc:163] Server-wide thread pool size limit: 3276
21:52:34.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:34.810607  9488 rpc_server.cc:307] RPC server started. Bound to: 127.9.28.252:40867
21:52:34.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:34.810614  9565 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.28.252:40867 every 8 connection(s)
21:52:34.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:34.814546  9488 server_base.cc:1179] Dumped server information to /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/master-2/data/info.pb
21:52:34.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:34.820729  9331 external_mini_cluster.cc:1428] Started /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu as pid 9488
21:52:34.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:34.821190  9331 external_mini_cluster.cc:1442] Reading /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/master-2/wal/instance
21:52:34.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:34.821938  9566 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:52:34.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:34.844030  9566 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.28.254" port: 37687 } has no permanent_uuid. Determining permanent_uuid...
21:52:34.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:34.873060  9566 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.28.253" port: 41061 } has no permanent_uuid. Determining permanent_uuid...
21:52:34.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:34.884305  9566 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.28.252" port: 40867 } has no permanent_uuid. Determining permanent_uuid...
21:52:34.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:34.902100  9566 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 574662fc86ca4395937a670f6394b8d0: Bootstrap starting.
21:52:34.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:34.910135  9566 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 574662fc86ca4395937a670f6394b8d0: Neither blocks nor log segments found. Creating new log.
21:52:34.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:34.912650  9566 log.cc:826] T 00000000000000000000000000000000 P 574662fc86ca4395937a670f6394b8d0: Log is configured to *not* fsync() on all Append() calls
21:52:34.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:34.917320  9566 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 574662fc86ca4395937a670f6394b8d0: No bootstrap required, opened a new log
21:52:34.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:34.938055  9566 raft_consensus.cc:357] T 00000000000000000000000000000000 P 574662fc86ca4395937a670f6394b8d0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d05473426f1e4e84a5fc4af172cc93f9" member_type: VOTER last_known_addr { host: "127.9.28.254" port: 37687 } } peers { permanent_uuid: "537228d41a4741d39050ca442dc448c9" member_type: VOTER last_known_addr { host: "127.9.28.253" port: 41061 } } peers { permanent_uuid: "574662fc86ca4395937a670f6394b8d0" member_type: VOTER last_known_addr { host: "127.9.28.252" port: 40867 } }
21:52:34.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:34.938730  9566 raft_consensus.cc:383] T 00000000000000000000000000000000 P 574662fc86ca4395937a670f6394b8d0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:52:34.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:34.938934  9566 raft_consensus.cc:738] T 00000000000000000000000000000000 P 574662fc86ca4395937a670f6394b8d0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 574662fc86ca4395937a670f6394b8d0, State: Initialized, Role: FOLLOWER
21:52:34.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:34.939705  9566 consensus_queue.cc:260] T 00000000000000000000000000000000 P 574662fc86ca4395937a670f6394b8d0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d05473426f1e4e84a5fc4af172cc93f9" member_type: VOTER last_known_addr { host: "127.9.28.254" port: 37687 } } peers { permanent_uuid: "537228d41a4741d39050ca442dc448c9" member_type: VOTER last_known_addr { host: "127.9.28.253" port: 41061 } } peers { permanent_uuid: "574662fc86ca4395937a670f6394b8d0" member_type: VOTER last_known_addr { host: "127.9.28.252" port: 40867 } }
21:52:34.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:34.942170  9575 sys_catalog.cc:455] T 00000000000000000000000000000000 P 574662fc86ca4395937a670f6394b8d0 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d05473426f1e4e84a5fc4af172cc93f9" member_type: VOTER last_known_addr { host: "127.9.28.254" port: 37687 } } peers { permanent_uuid: "537228d41a4741d39050ca442dc448c9" member_type: VOTER last_known_addr { host: "127.9.28.253" port: 41061 } } peers { permanent_uuid: "574662fc86ca4395937a670f6394b8d0" member_type: VOTER last_known_addr { host: "127.9.28.252" port: 40867 } } }
21:52:34.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:34.942873  9575 sys_catalog.cc:458] T 00000000000000000000000000000000 P 574662fc86ca4395937a670f6394b8d0 [sys.catalog]: This master's current role is: FOLLOWER
21:52:34.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:34.943847  9566 sys_catalog.cc:564] T 00000000000000000000000000000000 P 574662fc86ca4395937a670f6394b8d0 [sys.catalog]: configured and running, proceeding with master startup.
21:52:34.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:34.970362  9586 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 574662fc86ca4395937a670f6394b8d0: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:52:34.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:34.970695  9586 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:52:34.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:34.986187  9520 tablet.cc:2378] T 00000000000000000000000000000000 P 574662fc86ca4395937a670f6394b8d0: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:52:35.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:34.967067  9488 master_runner.cc:186] Error getting master registration for 127.9.28.254:37687: OK, instance_id { permanent_uuid: "d05473426f1e4e84a5fc4af172cc93f9" instance_seqno: 1756849950781061 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:52:35.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.001127  9488 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:52:35.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.202080  9575 raft_consensus.cc:491] T 00000000000000000000000000000000 P 574662fc86ca4395937a670f6394b8d0 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:52:35.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.202778  9486 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.28.252" port: 40867 } attempt: 7
21:52:35.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.202764  9575 raft_consensus.cc:513] T 00000000000000000000000000000000 P 574662fc86ca4395937a670f6394b8d0 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d05473426f1e4e84a5fc4af172cc93f9" member_type: VOTER last_known_addr { host: "127.9.28.254" port: 37687 } } peers { permanent_uuid: "537228d41a4741d39050ca442dc448c9" member_type: VOTER last_known_addr { host: "127.9.28.253" port: 41061 } } peers { permanent_uuid: "574662fc86ca4395937a670f6394b8d0" member_type: VOTER last_known_addr { host: "127.9.28.252" port: 40867 } }
21:52:35.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.207584  9575 leader_election.cc:290] T 00000000000000000000000000000000 P 574662fc86ca4395937a670f6394b8d0 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d05473426f1e4e84a5fc4af172cc93f9 (127.9.28.254:37687), 537228d41a4741d39050ca442dc448c9 (127.9.28.253:41061)
21:52:35.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.235605  9486 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 537228d41a4741d39050ca442dc448c9: Bootstrap starting.
21:52:35.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.250010  9486 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 537228d41a4741d39050ca442dc448c9: Neither blocks nor log segments found. Creating new log.
21:52:35.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.255049  9486 log.cc:826] T 00000000000000000000000000000000 P 537228d41a4741d39050ca442dc448c9: Log is configured to *not* fsync() on all Append() calls
21:52:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.262697  9486 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 537228d41a4741d39050ca442dc448c9: No bootstrap required, opened a new log
21:52:35.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.310775  9486 raft_consensus.cc:357] T 00000000000000000000000000000000 P 537228d41a4741d39050ca442dc448c9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d05473426f1e4e84a5fc4af172cc93f9" member_type: VOTER last_known_addr { host: "127.9.28.254" port: 37687 } } peers { permanent_uuid: "537228d41a4741d39050ca442dc448c9" member_type: VOTER last_known_addr { host: "127.9.28.253" port: 41061 } } peers { permanent_uuid: "574662fc86ca4395937a670f6394b8d0" member_type: VOTER last_known_addr { host: "127.9.28.252" port: 40867 } }
21:52:35.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.312772  9486 raft_consensus.cc:383] T 00000000000000000000000000000000 P 537228d41a4741d39050ca442dc448c9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:52:35.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.313506  9486 raft_consensus.cc:738] T 00000000000000000000000000000000 P 537228d41a4741d39050ca442dc448c9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 537228d41a4741d39050ca442dc448c9, State: Initialized, Role: FOLLOWER
21:52:35.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.315176  9486 consensus_queue.cc:260] T 00000000000000000000000000000000 P 537228d41a4741d39050ca442dc448c9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d05473426f1e4e84a5fc4af172cc93f9" member_type: VOTER last_known_addr { host: "127.9.28.254" port: 37687 } } peers { permanent_uuid: "537228d41a4741d39050ca442dc448c9" member_type: VOTER last_known_addr { host: "127.9.28.253" port: 41061 } } peers { permanent_uuid: "574662fc86ca4395937a670f6394b8d0" member_type: VOTER last_known_addr { host: "127.9.28.252" port: 40867 } }
21:52:35.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.319399  9590 sys_catalog.cc:455] T 00000000000000000000000000000000 P 537228d41a4741d39050ca442dc448c9 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d05473426f1e4e84a5fc4af172cc93f9" member_type: VOTER last_known_addr { host: "127.9.28.254" port: 37687 } } peers { permanent_uuid: "537228d41a4741d39050ca442dc448c9" member_type: VOTER last_known_addr { host: "127.9.28.253" port: 41061 } } peers { permanent_uuid: "574662fc86ca4395937a670f6394b8d0" member_type: VOTER last_known_addr { host: "127.9.28.252" port: 40867 } } }
21:52:35.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.320561  9590 sys_catalog.cc:458] T 00000000000000000000000000000000 P 537228d41a4741d39050ca442dc448c9 [sys.catalog]: This master's current role is: FOLLOWER
21:52:35.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.321624  9486 sys_catalog.cc:564] T 00000000000000000000000000000000 P 537228d41a4741d39050ca442dc448c9 [sys.catalog]: configured and running, proceeding with master startup.
21:52:35.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.321822  9461 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "574662fc86ca4395937a670f6394b8d0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "537228d41a4741d39050ca442dc448c9" is_pre_election: true
21:52:35.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.323078  9461 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 537228d41a4741d39050ca442dc448c9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 574662fc86ca4395937a670f6394b8d0 in term 0.
21:52:35.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.326269  9517 leader_election.cc:304] T 00000000000000000000000000000000 P 574662fc86ca4395937a670f6394b8d0 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 537228d41a4741d39050ca442dc448c9, 574662fc86ca4395937a670f6394b8d0; no voters: 
21:52:35.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.327967  9575 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 574662fc86ca4395937a670f6394b8d0 [term 0 FOLLOWER]: Leader pre-election won for term 1
21:52:35.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.328370  9575 raft_consensus.cc:491] T 00000000000000000000000000000000 P 574662fc86ca4395937a670f6394b8d0 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:52:35.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.328706  9575 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 574662fc86ca4395937a670f6394b8d0 [term 0 FOLLOWER]: Advancing to term 1
21:52:35.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.339589  9575 raft_consensus.cc:513] T 00000000000000000000000000000000 P 574662fc86ca4395937a670f6394b8d0 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d05473426f1e4e84a5fc4af172cc93f9" member_type: VOTER last_known_addr { host: "127.9.28.254" port: 37687 } } peers { permanent_uuid: "537228d41a4741d39050ca442dc448c9" member_type: VOTER last_known_addr { host: "127.9.28.253" port: 41061 } } peers { permanent_uuid: "574662fc86ca4395937a670f6394b8d0" member_type: VOTER last_known_addr { host: "127.9.28.252" port: 40867 } }
21:52:35.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.352346  9461 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "574662fc86ca4395937a670f6394b8d0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "537228d41a4741d39050ca442dc448c9"
21:52:35.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.352988  9461 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 537228d41a4741d39050ca442dc448c9 [term 0 FOLLOWER]: Advancing to term 1
21:52:35.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.363430  9575 leader_election.cc:290] T 00000000000000000000000000000000 P 574662fc86ca4395937a670f6394b8d0 [CANDIDATE]: Term 1 election: Requested vote from peers d05473426f1e4e84a5fc4af172cc93f9 (127.9.28.254:37687), 537228d41a4741d39050ca442dc448c9 (127.9.28.253:41061)
21:52:35.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.366861  9461 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 537228d41a4741d39050ca442dc448c9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 574662fc86ca4395937a670f6394b8d0 in term 1.
21:52:35.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.369040  9517 leader_election.cc:304] T 00000000000000000000000000000000 P 574662fc86ca4395937a670f6394b8d0 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 537228d41a4741d39050ca442dc448c9, 574662fc86ca4395937a670f6394b8d0; no voters: 
21:52:35.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.370803  9575 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 574662fc86ca4395937a670f6394b8d0 [term 1 FOLLOWER]: Leader election won for term 1
21:52:35.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.373937  9575 raft_consensus.cc:695] T 00000000000000000000000000000000 P 574662fc86ca4395937a670f6394b8d0 [term 1 LEADER]: Becoming Leader. State: Replica: 574662fc86ca4395937a670f6394b8d0, State: Running, Role: LEADER
21:52:35.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.375031  9575 consensus_queue.cc:237] T 00000000000000000000000000000000 P 574662fc86ca4395937a670f6394b8d0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d05473426f1e4e84a5fc4af172cc93f9" member_type: VOTER last_known_addr { host: "127.9.28.254" port: 37687 } } peers { permanent_uuid: "537228d41a4741d39050ca442dc448c9" member_type: VOTER last_known_addr { host: "127.9.28.253" port: 41061 } } peers { permanent_uuid: "574662fc86ca4395937a670f6394b8d0" member_type: VOTER last_known_addr { host: "127.9.28.252" port: 40867 } }
21:52:35.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.394439  9599 sys_catalog.cc:455] T 00000000000000000000000000000000 P 574662fc86ca4395937a670f6394b8d0 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 574662fc86ca4395937a670f6394b8d0. Latest consensus state: current_term: 1 leader_uuid: "574662fc86ca4395937a670f6394b8d0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d05473426f1e4e84a5fc4af172cc93f9" member_type: VOTER last_known_addr { host: "127.9.28.254" port: 37687 } } peers { permanent_uuid: "537228d41a4741d39050ca442dc448c9" member_type: VOTER last_known_addr { host: "127.9.28.253" port: 41061 } } peers { permanent_uuid: "574662fc86ca4395937a670f6394b8d0" member_type: VOTER last_known_addr { host: "127.9.28.252" port: 40867 } } }
21:52:35.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.395279  9599 sys_catalog.cc:458] T 00000000000000000000000000000000 P 574662fc86ca4395937a670f6394b8d0 [sys.catalog]: This master's current role is: LEADER
21:52:35.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.398943  9604 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
21:52:35.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.413518  9604 catalog_manager.cc:1486] Initializing Kudu cluster ID...
21:52:35.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:35.425911  9603 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 537228d41a4741d39050ca442dc448c9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:52:35.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:35.426682  9603 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:52:35.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.454393  9461 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 537228d41a4741d39050ca442dc448c9 [term 1 FOLLOWER]: Refusing update from remote peer 574662fc86ca4395937a670f6394b8d0: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:52:35.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.456749  9599 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 574662fc86ca4395937a670f6394b8d0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "537228d41a4741d39050ca442dc448c9" member_type: VOTER last_known_addr { host: "127.9.28.253" port: 41061 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
21:52:35.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.477079  9411 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.28.252" port: 40867 } attempt: 7
21:52:35.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:35.491271  9440 tablet.cc:2378] T 00000000000000000000000000000000 P 537228d41a4741d39050ca442dc448c9: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:52:35.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.406625  9413 master_runner.cc:186] Error getting master registration for 127.9.28.254:37687: OK, instance_id { permanent_uuid: "d05473426f1e4e84a5fc4af172cc93f9" instance_seqno: 1756849950781061 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:52:35.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.512377  9413 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:52:35.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.541921  9590 sys_catalog.cc:455] T 00000000000000000000000000000000 P 537228d41a4741d39050ca442dc448c9 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 574662fc86ca4395937a670f6394b8d0. Latest consensus state: current_term: 1 leader_uuid: "574662fc86ca4395937a670f6394b8d0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d05473426f1e4e84a5fc4af172cc93f9" member_type: VOTER last_known_addr { host: "127.9.28.254" port: 37687 } } peers { permanent_uuid: "537228d41a4741d39050ca442dc448c9" member_type: VOTER last_known_addr { host: "127.9.28.253" port: 41061 } } peers { permanent_uuid: "574662fc86ca4395937a670f6394b8d0" member_type: VOTER last_known_addr { host: "127.9.28.252" port: 40867 } } }
21:52:35.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.542790  9590 sys_catalog.cc:458] T 00000000000000000000000000000000 P 537228d41a4741d39050ca442dc448c9 [sys.catalog]: This master's current role is: FOLLOWER
21:52:35.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.546554  9411 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d05473426f1e4e84a5fc4af172cc93f9: Bootstrap starting.
21:52:35.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.555392  9608 mvcc.cc:204] Tried to move back new op lower bound from 7196057417520332800 to 7196057417229099008. Current Snapshot: MvccSnapshot[applied={T|T < 7196057417520332800}]
21:52:35.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.556799  9575 sys_catalog.cc:455] T 00000000000000000000000000000000 P 574662fc86ca4395937a670f6394b8d0 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "574662fc86ca4395937a670f6394b8d0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d05473426f1e4e84a5fc4af172cc93f9" member_type: VOTER last_known_addr { host: "127.9.28.254" port: 37687 } } peers { permanent_uuid: "537228d41a4741d39050ca442dc448c9" member_type: VOTER last_known_addr { host: "127.9.28.253" port: 41061 } } peers { permanent_uuid: "574662fc86ca4395937a670f6394b8d0" member_type: VOTER last_known_addr { host: "127.9.28.252" port: 40867 } } }
21:52:35.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.558400  9411 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P d05473426f1e4e84a5fc4af172cc93f9: Neither blocks nor log segments found. Creating new log.
21:52:35.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.558413  9575 sys_catalog.cc:458] T 00000000000000000000000000000000 P 574662fc86ca4395937a670f6394b8d0 [sys.catalog]: This master's current role is: LEADER
21:52:35.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.561573  9604 catalog_manager.cc:1349] Generated new cluster ID: dedcb1106cfe408fb2771623ff5ff475
21:52:35.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.561899  9604 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
21:52:35.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.566701  9590 sys_catalog.cc:455] T 00000000000000000000000000000000 P 537228d41a4741d39050ca442dc448c9 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "574662fc86ca4395937a670f6394b8d0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d05473426f1e4e84a5fc4af172cc93f9" member_type: VOTER last_known_addr { host: "127.9.28.254" port: 37687 } } peers { permanent_uuid: "537228d41a4741d39050ca442dc448c9" member_type: VOTER last_known_addr { host: "127.9.28.253" port: 41061 } } peers { permanent_uuid: "574662fc86ca4395937a670f6394b8d0" member_type: VOTER last_known_addr { host: "127.9.28.252" port: 40867 } } }
21:52:35.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.567489  9590 sys_catalog.cc:458] T 00000000000000000000000000000000 P 537228d41a4741d39050ca442dc448c9 [sys.catalog]: This master's current role is: FOLLOWER
21:52:35.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.570222  9411 log.cc:826] T 00000000000000000000000000000000 P d05473426f1e4e84a5fc4af172cc93f9: Log is configured to *not* fsync() on all Append() calls
21:52:35.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.597456  9411 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d05473426f1e4e84a5fc4af172cc93f9: No bootstrap required, opened a new log
21:52:35.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.625654  9604 catalog_manager.cc:1372] Generated new certificate authority record
21:52:35.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.628510  9604 catalog_manager.cc:1506] Loading token signing keys...
21:52:35.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.662812  9411 raft_consensus.cc:357] T 00000000000000000000000000000000 P d05473426f1e4e84a5fc4af172cc93f9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d05473426f1e4e84a5fc4af172cc93f9" member_type: VOTER last_known_addr { host: "127.9.28.254" port: 37687 } } peers { permanent_uuid: "537228d41a4741d39050ca442dc448c9" member_type: VOTER last_known_addr { host: "127.9.28.253" port: 41061 } } peers { permanent_uuid: "574662fc86ca4395937a670f6394b8d0" member_type: VOTER last_known_addr { host: "127.9.28.252" port: 40867 } }
21:52:35.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.670267  9411 raft_consensus.cc:383] T 00000000000000000000000000000000 P d05473426f1e4e84a5fc4af172cc93f9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:52:35.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.671000  9411 raft_consensus.cc:738] T 00000000000000000000000000000000 P d05473426f1e4e84a5fc4af172cc93f9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d05473426f1e4e84a5fc4af172cc93f9, State: Initialized, Role: FOLLOWER
21:52:35.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.677052  9411 consensus_queue.cc:260] T 00000000000000000000000000000000 P d05473426f1e4e84a5fc4af172cc93f9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d05473426f1e4e84a5fc4af172cc93f9" member_type: VOTER last_known_addr { host: "127.9.28.254" port: 37687 } } peers { permanent_uuid: "537228d41a4741d39050ca442dc448c9" member_type: VOTER last_known_addr { host: "127.9.28.253" port: 41061 } } peers { permanent_uuid: "574662fc86ca4395937a670f6394b8d0" member_type: VOTER last_known_addr { host: "127.9.28.252" port: 40867 } }
21:52:35.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.682111  9613 sys_catalog.cc:455] T 00000000000000000000000000000000 P d05473426f1e4e84a5fc4af172cc93f9 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d05473426f1e4e84a5fc4af172cc93f9" member_type: VOTER last_known_addr { host: "127.9.28.254" port: 37687 } } peers { permanent_uuid: "537228d41a4741d39050ca442dc448c9" member_type: VOTER last_known_addr { host: "127.9.28.253" port: 41061 } } peers { permanent_uuid: "574662fc86ca4395937a670f6394b8d0" member_type: VOTER last_known_addr { host: "127.9.28.252" port: 40867 } } }
21:52:35.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.684639  9613 sys_catalog.cc:458] T 00000000000000000000000000000000 P d05473426f1e4e84a5fc4af172cc93f9 [sys.catalog]: This master's current role is: FOLLOWER
21:52:35.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.692641  9411 sys_catalog.cc:564] T 00000000000000000000000000000000 P d05473426f1e4e84a5fc4af172cc93f9 [sys.catalog]: configured and running, proceeding with master startup.
21:52:35.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.696125  9386 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "574662fc86ca4395937a670f6394b8d0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d05473426f1e4e84a5fc4af172cc93f9" is_pre_election: true
21:52:35.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.697167  9386 raft_consensus.cc:2466] T 00000000000000000000000000000000 P d05473426f1e4e84a5fc4af172cc93f9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 574662fc86ca4395937a670f6394b8d0 in term 0.
21:52:35.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.698906  9384 raft_consensus.cc:3058] T 00000000000000000000000000000000 P d05473426f1e4e84a5fc4af172cc93f9 [term 0 FOLLOWER]: Advancing to term 1
21:52:35.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.702976  9385 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "574662fc86ca4395937a670f6394b8d0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d05473426f1e4e84a5fc4af172cc93f9"
21:52:35.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.719358  9604 catalog_manager.cc:5955] T 00000000000000000000000000000000 P 574662fc86ca4395937a670f6394b8d0: Generated new TSK 0
21:52:35.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.720744  9604 catalog_manager.cc:1516] Initializing in-progress tserver states...
21:52:35.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.723985  9384 raft_consensus.cc:1273] T 00000000000000000000000000000000 P d05473426f1e4e84a5fc4af172cc93f9 [term 1 FOLLOWER]: Refusing update from remote peer 574662fc86ca4395937a670f6394b8d0: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:52:35.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.726313  9575 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 574662fc86ca4395937a670f6394b8d0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d05473426f1e4e84a5fc4af172cc93f9" member_type: VOTER last_known_addr { host: "127.9.28.254" port: 37687 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
21:52:35.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:35.736063  9365 tablet.cc:2378] T 00000000000000000000000000000000 P d05473426f1e4e84a5fc4af172cc93f9: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:52:35.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.757229  9331 external_mini_cluster.cc:1366] Running /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu
21:52:35.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu
21:52:35.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/ts-0/wal
21:52:35.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/ts-0/data
21:52:35.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:52:35.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:52:35.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:52:35.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:52:35.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:52:35.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:52:35.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/ts-0/logs
21:52:35.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/ts-0/data/info.pb
21:52:35.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:52:35.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:52:35.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:52:35.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:52:35.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:52:35.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:52:35.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:52:35.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:52:35.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:52:35.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:52:35.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.28.193:0
21:52:35.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.28.193
21:52:35.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.28.193
21:52:35.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:52:35.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.28.254:37687,127.9.28.253:41061,127.9.28.252:40867
21:52:35.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.28.212:34313
21:52:35.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:52:35.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:52:35.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:52:35.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.798592  9613 sys_catalog.cc:455] T 00000000000000000000000000000000 P d05473426f1e4e84a5fc4af172cc93f9 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 574662fc86ca4395937a670f6394b8d0. Latest consensus state: current_term: 1 leader_uuid: "574662fc86ca4395937a670f6394b8d0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d05473426f1e4e84a5fc4af172cc93f9" member_type: VOTER last_known_addr { host: "127.9.28.254" port: 37687 } } peers { permanent_uuid: "537228d41a4741d39050ca442dc448c9" member_type: VOTER last_known_addr { host: "127.9.28.253" port: 41061 } } peers { permanent_uuid: "574662fc86ca4395937a670f6394b8d0" member_type: VOTER last_known_addr { host: "127.9.28.252" port: 40867 } } }
21:52:35.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.798617  9622 sys_catalog.cc:455] T 00000000000000000000000000000000 P d05473426f1e4e84a5fc4af172cc93f9 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "574662fc86ca4395937a670f6394b8d0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d05473426f1e4e84a5fc4af172cc93f9" member_type: VOTER last_known_addr { host: "127.9.28.254" port: 37687 } } peers { permanent_uuid: "537228d41a4741d39050ca442dc448c9" member_type: VOTER last_known_addr { host: "127.9.28.253" port: 41061 } } peers { permanent_uuid: "574662fc86ca4395937a670f6394b8d0" member_type: VOTER last_known_addr { host: "127.9.28.252" port: 40867 } } }
21:52:35.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.799471  9613 sys_catalog.cc:458] T 00000000000000000000000000000000 P d05473426f1e4e84a5fc4af172cc93f9 [sys.catalog]: This master's current role is: FOLLOWER
21:52:35.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.799571  9622 sys_catalog.cc:458] T 00000000000000000000000000000000 P d05473426f1e4e84a5fc4af172cc93f9 [sys.catalog]: This master's current role is: FOLLOWER
21:52:35.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.875517  9630 catalog_manager.cc:1261] Loaded cluster ID: dedcb1106cfe408fb2771623ff5ff475
21:52:35.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.876044  9630 catalog_manager.cc:1554] T 00000000000000000000000000000000 P d05473426f1e4e84a5fc4af172cc93f9: loading cluster ID for follower catalog manager: success
21:52:35.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.916046  9630 catalog_manager.cc:1576] T 00000000000000000000000000000000 P d05473426f1e4e84a5fc4af172cc93f9: acquiring CA information for follower catalog manager: success
21:52:35.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:35.921813  9630 catalog_manager.cc:1604] T 00000000000000000000000000000000 P d05473426f1e4e84a5fc4af172cc93f9: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:52:36.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:36.302677  9575 sys_catalog.cc:455] T 00000000000000000000000000000000 P 574662fc86ca4395937a670f6394b8d0 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "574662fc86ca4395937a670f6394b8d0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d05473426f1e4e84a5fc4af172cc93f9" member_type: VOTER last_known_addr { host: "127.9.28.254" port: 37687 } } peers { permanent_uuid: "537228d41a4741d39050ca442dc448c9" member_type: VOTER last_known_addr { host: "127.9.28.253" port: 41061 } } peers { permanent_uuid: "574662fc86ca4395937a670f6394b8d0" member_type: VOTER last_known_addr { host: "127.9.28.252" port: 40867 } } }
21:52:36.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:36.304200  9575 sys_catalog.cc:458] T 00000000000000000000000000000000 P 574662fc86ca4395937a670f6394b8d0 [sys.catalog]: This master's current role is: LEADER
21:52:36.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:36.401543  9623 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:52:36.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:36.402554  9623 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:52:36.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:36.403653  9623 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:52:36.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:36.432660  9603 catalog_manager.cc:1261] Loaded cluster ID: dedcb1106cfe408fb2771623ff5ff475
21:52:36.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:36.433190  9603 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 537228d41a4741d39050ca442dc448c9: loading cluster ID for follower catalog manager: success
21:52:36.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:36.440999  9603 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 537228d41a4741d39050ca442dc448c9: acquiring CA information for follower catalog manager: success
21:52:36.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:36.457024  9603 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 537228d41a4741d39050ca442dc448c9: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:52:36.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:36.467406  9623 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:52:36.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:36.468911  9623 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.9.28.193
21:52:36.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:36.540798  9623 tablet_server_runner.cc:78] Tablet server non-default flags:
21:52:36.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:52:36.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.28.212:34313
21:52:36.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:52:36.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:52:36.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/ts-0/data
21:52:36.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/ts-0/wal
21:52:36.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:52:36.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:52:36.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.28.193:0
21:52:36.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:52:36.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:52:36.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:52:36.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/ts-0/data/info.pb
21:52:36.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.28.193
21:52:36.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:52:36.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.28.254:37687,127.9.28.253:41061,127.9.28.252:40867
21:52:36.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:52:36.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:52:36.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:52:36.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:52:36.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:52:36.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.28.193
21:52:36.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/ts-0/logs
21:52:36.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:52:36.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:52:36.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:52:36.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:52:36.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:52:36.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6ab3f249351a4ee5d73eb8bda3585c4dcd85d070
21:52:36.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:52:36.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 21:30:03 UTC on 5fd53c4cbb9d
21:52:36.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7869
21:52:36.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:52:36.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:36.542578  9623 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:52:36.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:36.545115  9623 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:52:36.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:36.582355  9640 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:52:38.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:37.968521  9639 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ea0 after lost signal to thread 9623
21:52:38.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:38.939096  9639 kernel_stack_watchdog.cc:198] Thread 9623 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 382ms:
21:52:38.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
21:52:38.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
21:52:38.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
21:52:38.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
21:52:38.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:38.939357  9623 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 2.355s	user 0.611s	sys 1.125s
21:52:38.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:38.939833  9623 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 2.355s	user 0.611s	sys 1.125s
21:52:38.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:36.586956  9641 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:52:38.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:38.950415  9646 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:52:38.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:38.952175  9562 debug-util.cc:398] Leaking SignalData structure 0x7b080003bea0 after lost signal to thread 9493
21:52:38.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:38.952987  9562 debug-util.cc:398] Leaking SignalData structure 0x7b08000976e0 after lost signal to thread 9565
21:52:38.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:38.960711  9623 server_base.cc:1047] running on GCE node
21:52:38.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:38.962175  9623 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:52:38.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:38.973191  9623 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:52:38.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:38.981058  9623 hybrid_clock.cc:648] HybridClock initialized: now 1756849958980948 us; error 111 us; skew 500 ppm
21:52:38.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:38.982580  9623 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:52:39.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:39.011857  9623 webserver.cc:480] Webserver started at http://127.9.28.193:35443/ using document root <none> and password file <none>
21:52:39.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:39.013237  9623 fs_manager.cc:362] Metadata directory not provided
21:52:39.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:39.013681  9623 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:52:39.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:39.014377  9623 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:52:39.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:39.021364  9623 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/ts-0/data/instance:
21:52:39.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7e4fa962a5bc461093583bd167d50dea"
21:52:39.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:52:39 on dist-test-slave-q7qm"
21:52:39.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:39.030443  9623 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/ts-0/wal/instance:
21:52:39.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7e4fa962a5bc461093583bd167d50dea"
21:52:39.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:52:39 on dist-test-slave-q7qm"
21:52:39.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:39.056077  9623 fs_manager.cc:696] Time spent creating directory manager: real 0.025s	user 0.013s	sys 0.012s
21:52:39.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:39.075870  9651 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:52:39.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:39.077795  9623 fs_manager.cc:730] Time spent opening block manager: real 0.016s	user 0.005s	sys 0.005s
21:52:39.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:39.078301  9623 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/ts-0/data,/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/ts-0/wal
21:52:39.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7e4fa962a5bc461093583bd167d50dea"
21:52:39.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:52:39 on dist-test-slave-q7qm"
21:52:39.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:39.078827  9623 fs_report.cc:389] FS layout report
21:52:39.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:52:39.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/ts-0/wal
21:52:39.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/ts-0/wal
21:52:39.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/ts-0/data/data
21:52:39.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:52:39.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:52:39.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:52:39.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:52:39.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:52:39.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:52:39.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:52:39.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:52:39.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:52:39.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:52:39.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:52:39.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:39.198379  9623 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:52:39.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:39.200866  9623 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:52:39.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:39.201907  9623 kserver.cc:163] Server-wide thread pool size limit: 3276
21:52:39.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:39.212877  9623 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:52:39.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:39.225849  9623 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:52:39.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:39.226188  9623 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
21:52:39.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:39.226568  9623 ts_tablet_manager.cc:610] Registered 0 tablets
21:52:39.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:39.226814  9623 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
21:52:39.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:39.729581  9623 rpc_server.cc:307] RPC server started. Bound to: 127.9.28.193:33195
21:52:39.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:39.729718  9764 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.28.193:33195 every 8 connection(s)
21:52:39.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:39.734624  9623 server_base.cc:1179] Dumped server information to /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/ts-0/data/info.pb
21:52:39.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:39.744586  9331 external_mini_cluster.cc:1428] Started /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu as pid 9623
21:52:39.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:39.745267  9331 external_mini_cluster.cc:1442] Reading /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/ts-0/wal/instance
21:52:39.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:39.762410  9331 external_mini_cluster.cc:1366] Running /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu
21:52:39.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu
21:52:39.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/ts-1/wal
21:52:39.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/ts-1/data
21:52:39.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:52:39.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:52:39.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:52:39.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:52:39.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:52:39.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:52:39.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/ts-1/logs
21:52:39.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/ts-1/data/info.pb
21:52:39.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:52:39.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:52:39.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:52:39.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:52:39.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:52:39.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:52:39.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:52:39.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:52:39.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:52:39.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:52:39.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.28.194:0
21:52:39.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.28.194
21:52:39.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.28.194
21:52:39.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:52:39.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.28.254:37687,127.9.28.253:41061,127.9.28.252:40867
21:52:39.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.28.212:34313
21:52:39.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:52:39.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:52:39.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:52:39.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:39.850936  9765 heartbeater.cc:344] Connected to a master server at 127.9.28.252:40867
21:52:39.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:39.851686  9765 heartbeater.cc:461] Registering TS with master...
21:52:39.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:39.853179  9765 heartbeater.cc:507] Master 127.9.28.252:40867 requested a full tablet report, sending...
21:52:39.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:39.854879  9766 heartbeater.cc:344] Connected to a master server at 127.9.28.253:41061
21:52:39.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:39.855315  9766 heartbeater.cc:461] Registering TS with master...
21:52:39.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:39.856350  9766 heartbeater.cc:507] Master 127.9.28.253:41061 requested a full tablet report, sending...
21:52:39.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:39.860633  9531 ts_manager.cc:194] Registered new tserver with Master: 7e4fa962a5bc461093583bd167d50dea (127.9.28.193:33195)
21:52:39.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:39.860800  9451 ts_manager.cc:194] Registered new tserver with Master: 7e4fa962a5bc461093583bd167d50dea (127.9.28.193:33195)
21:52:39.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:39.866142  9531 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.9.28.193:59769
21:52:39.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:39.882458  9770 heartbeater.cc:344] Connected to a master server at 127.9.28.254:37687
21:52:39.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:39.882920  9770 heartbeater.cc:461] Registering TS with master...
21:52:39.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:39.883746  9770 heartbeater.cc:507] Master 127.9.28.254:37687 requested a full tablet report, sending...
21:52:39.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:39.888057  9376 ts_manager.cc:194] Registered new tserver with Master: 7e4fa962a5bc461093583bd167d50dea (127.9.28.193:33195)
21:52:40.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:40.272074  9769 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:52:40.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:40.272574  9769 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:52:40.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:40.273039  9769 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:52:40.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:40.311405  9769 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:52:40.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:40.312318  9769 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.9.28.194
21:52:40.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:40.354338  9769 tablet_server_runner.cc:78] Tablet server non-default flags:
21:52:40.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:52:40.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.28.212:34313
21:52:40.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:52:40.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:52:40.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/ts-1/data
21:52:40.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/ts-1/wal
21:52:40.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:52:40.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:52:40.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.28.194:0
21:52:40.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:52:40.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:52:40.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:52:40.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/ts-1/data/info.pb
21:52:40.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.28.194
21:52:40.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:52:40.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.28.254:37687,127.9.28.253:41061,127.9.28.252:40867
21:52:40.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:52:40.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:52:40.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:52:40.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:52:40.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:52:40.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.28.194
21:52:40.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/ts-1/logs
21:52:40.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:52:40.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:52:40.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:52:40.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:52:40.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:52:40.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6ab3f249351a4ee5d73eb8bda3585c4dcd85d070
21:52:40.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:52:40.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 21:30:03 UTC on 5fd53c4cbb9d
21:52:40.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7869
21:52:40.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:52:40.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:40.355749  9769 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:52:40.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:40.357481  9769 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:52:40.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:40.376036  9782 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:52:40.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:40.871342  9765 heartbeater.cc:499] Master 127.9.28.252:40867 was elected leader, sending a full tablet report...
21:52:42.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:41.776305  9780 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ea0 after lost signal to thread 9769
21:52:42.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:42.358197  9783 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1982 milliseconds
21:52:42.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:40.376158  9781 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:52:42.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:42.357144  9769 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.983s	user 0.000s	sys 0.002s
21:52:42.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:42.358682  9769 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.984s	user 0.000s	sys 0.002s
21:52:42.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:42.361333  9784 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:52:42.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:42.361387  9769 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:52:42.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:42.372747  9769 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:52:42.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:42.375564  9769 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:52:42.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:42.377074  9769 hybrid_clock.cc:648] HybridClock initialized: now 1756849962377020 us; error 64 us; skew 500 ppm
21:52:42.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:42.377954  9769 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:52:42.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:42.396754  9769 webserver.cc:480] Webserver started at http://127.9.28.194:37873/ using document root <none> and password file <none>
21:52:42.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:42.398159  9769 fs_manager.cc:362] Metadata directory not provided
21:52:42.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:42.398512  9769 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:52:42.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:42.399099  9769 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:52:42.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:42.409323  9769 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/ts-1/data/instance:
21:52:42.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7c0f642d41e0456d9ee7a97d736f573e"
21:52:42.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:52:42 on dist-test-slave-q7qm"
21:52:42.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:42.411242  9769 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/ts-1/wal/instance:
21:52:42.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7c0f642d41e0456d9ee7a97d736f573e"
21:52:42.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:52:42 on dist-test-slave-q7qm"
21:52:42.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:42.423345  9769 fs_manager.cc:696] Time spent creating directory manager: real 0.011s	user 0.008s	sys 0.001s
21:52:42.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:42.440402  9792 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:52:42.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:42.446980  9769 fs_manager.cc:730] Time spent opening block manager: real 0.014s	user 0.001s	sys 0.004s
21:52:42.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:42.447501  9769 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/ts-1/data,/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/ts-1/wal
21:52:42.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7c0f642d41e0456d9ee7a97d736f573e"
21:52:42.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:52:42 on dist-test-slave-q7qm"
21:52:42.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:42.447889  9769 fs_report.cc:389] FS layout report
21:52:42.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:52:42.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/ts-1/wal
21:52:42.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/ts-1/wal
21:52:42.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/ts-1/data/data
21:52:42.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:52:42.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:52:42.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:52:42.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:52:42.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:52:42.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:52:42.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:52:42.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:52:42.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:52:42.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:52:42.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:52:42.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:42.554196  9769 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:52:42.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:42.555891  9769 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:52:42.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:42.556455  9769 kserver.cc:163] Server-wide thread pool size limit: 3276
21:52:42.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:42.559885  9769 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:52:42.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:42.566593  9769 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:52:42.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:42.566835  9769 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
21:52:42.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:42.567039  9769 ts_tablet_manager.cc:610] Registered 0 tablets
21:52:42.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:42.567160  9769 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
21:52:42.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:42.734889  9769 rpc_server.cc:307] RPC server started. Bound to: 127.9.28.194:34061
21:52:42.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:42.734964  9904 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.28.194:34061 every 8 connection(s)
21:52:42.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:42.738859  9769 server_base.cc:1179] Dumped server information to /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/ts-1/data/info.pb
21:52:42.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:42.743084  9331 external_mini_cluster.cc:1428] Started /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu as pid 9769
21:52:42.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:42.743616  9331 external_mini_cluster.cc:1442] Reading /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/ts-1/wal/instance
21:52:42.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:42.771274  9331 external_mini_cluster.cc:1366] Running /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu
21:52:42.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu
21:52:42.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/ts-2/wal
21:52:42.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/ts-2/data
21:52:42.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:52:42.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:52:42.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:52:42.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:52:42.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:52:42.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:52:42.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/ts-2/logs
21:52:42.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/ts-2/data/info.pb
21:52:42.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:52:42.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:52:42.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:52:42.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:52:42.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:52:42.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:52:42.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:52:42.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:52:42.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:52:42.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:52:42.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.28.195:0
21:52:42.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.28.195
21:52:42.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.28.195
21:52:42.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:52:42.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.28.254:37687,127.9.28.253:41061,127.9.28.252:40867
21:52:42.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.28.212:34313
21:52:42.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:52:42.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:52:42.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:52:42.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:42.803808  9905 heartbeater.cc:344] Connected to a master server at 127.9.28.252:40867
21:52:42.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:42.804000  9909 heartbeater.cc:344] Connected to a master server at 127.9.28.254:37687
21:52:42.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:42.804586  9905 heartbeater.cc:461] Registering TS with master...
21:52:42.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:42.804582  9909 heartbeater.cc:461] Registering TS with master...
21:52:42.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:42.806944  9905 heartbeater.cc:507] Master 127.9.28.252:40867 requested a full tablet report, sending...
21:52:42.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:42.807152  9906 heartbeater.cc:344] Connected to a master server at 127.9.28.253:41061
21:52:42.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:42.807791  9906 heartbeater.cc:461] Registering TS with master...
21:52:42.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:42.809293  9906 heartbeater.cc:507] Master 127.9.28.253:41061 requested a full tablet report, sending...
21:52:42.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:42.811491  9531 ts_manager.cc:194] Registered new tserver with Master: 7c0f642d41e0456d9ee7a97d736f573e (127.9.28.194:34061)
21:52:42.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:42.812141  9909 heartbeater.cc:507] Master 127.9.28.254:37687 requested a full tablet report, sending...
21:52:42.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:42.812494  9451 ts_manager.cc:194] Registered new tserver with Master: 7c0f642d41e0456d9ee7a97d736f573e (127.9.28.194:34061)
21:52:42.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:42.813258  9531 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.9.28.194:57069
21:52:42.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:42.815003  9376 ts_manager.cc:194] Registered new tserver with Master: 7c0f642d41e0456d9ee7a97d736f573e (127.9.28.194:34061)
21:52:43.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:43.160516  9915 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:52:43.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:43.161038  9915 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:52:43.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:43.161525  9915 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:52:43.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:43.197644  9915 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:52:43.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:43.198490  9915 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.9.28.195
21:52:43.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:43.236377  9915 tablet_server_runner.cc:78] Tablet server non-default flags:
21:52:43.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:52:43.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.28.212:34313
21:52:43.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:52:43.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:52:43.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/ts-2/data
21:52:43.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/ts-2/wal
21:52:43.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:52:43.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:52:43.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.28.195:0
21:52:43.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:52:43.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:52:43.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:52:43.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/ts-2/data/info.pb
21:52:43.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.28.195
21:52:43.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:52:43.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.28.254:37687,127.9.28.253:41061,127.9.28.252:40867
21:52:43.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:52:43.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:52:43.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:52:43.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:52:43.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:52:43.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.28.195
21:52:43.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/ts-2/logs
21:52:43.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:52:43.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:52:43.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:52:43.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:52:43.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:52:43.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6ab3f249351a4ee5d73eb8bda3585c4dcd85d070
21:52:43.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:52:43.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 21:30:03 UTC on 5fd53c4cbb9d
21:52:43.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7869
21:52:43.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:52:43.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:43.237746  9915 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:52:43.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:43.239511  9915 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:52:43.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:43.253859  9922 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:52:43.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:43.455485  9407 debug-util.cc:398] Leaking SignalData structure 0x7b08000a72e0 after lost signal to thread 9344
21:52:43.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:43.817729  9905 heartbeater.cc:499] Master 127.9.28.252:40867 was elected leader, sending a full tablet report...
21:52:44.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:43.264247  9925 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:52:44.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:43.255169  9923 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:52:44.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:44.693604  9924 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1433 milliseconds
21:52:44.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:44.693811  9915 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:52:44.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:44.695612  9915 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:52:44.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:44.698881  9915 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:52:44.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:44.700578  9915 hybrid_clock.cc:648] HybridClock initialized: now 1756849964700531 us; error 44 us; skew 500 ppm
21:52:44.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:44.701990  9915 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:52:44.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:44.710975  9915 webserver.cc:480] Webserver started at http://127.9.28.195:36253/ using document root <none> and password file <none>
21:52:44.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:44.712707  9915 fs_manager.cc:362] Metadata directory not provided
21:52:44.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:44.713050  9915 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:52:44.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:44.713689  9915 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:52:44.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:44.721416  9915 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/ts-2/data/instance:
21:52:44.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "245563a49aa14536a93528a012b68226"
21:52:44.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:52:44 on dist-test-slave-q7qm"
21:52:44.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:44.723578  9915 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/ts-2/wal/instance:
21:52:44.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "245563a49aa14536a93528a012b68226"
21:52:44.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:52:44 on dist-test-slave-q7qm"
21:52:44.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:44.741079  9915 fs_manager.cc:696] Time spent creating directory manager: real 0.016s	user 0.014s	sys 0.001s
21:52:44.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:44.750986  9932 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:52:44.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:44.752589  9915 fs_manager.cc:730] Time spent opening block manager: real 0.006s	user 0.006s	sys 0.001s
21:52:44.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:44.753018  9915 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/ts-2/data,/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/ts-2/wal
21:52:44.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "245563a49aa14536a93528a012b68226"
21:52:44.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:52:44 on dist-test-slave-q7qm"
21:52:44.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:44.753387  9915 fs_report.cc:389] FS layout report
21:52:44.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:52:44.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/ts-2/wal
21:52:44.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/ts-2/wal
21:52:44.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/ts-2/data/data
21:52:44.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:52:44.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:52:44.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:52:44.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:52:44.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:52:44.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:52:44.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:52:44.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:52:44.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:52:44.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:52:44.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:52:44.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:44.825059  9915 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:52:44.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:44.827183  9915 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:52:44.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:44.827836  9915 kserver.cc:163] Server-wide thread pool size limit: 3276
21:52:44.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:44.832335  9915 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:52:44.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:44.842859  9915 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:52:44.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:44.843200  9915 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
21:52:44.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:44.843529  9915 ts_tablet_manager.cc:610] Registered 0 tablets
21:52:44.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:44.843889  9915 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
21:52:45.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:45.041555  9915 rpc_server.cc:307] RPC server started. Bound to: 127.9.28.195:39395
21:52:45.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:45.041606 10044 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.28.195:39395 every 8 connection(s)
21:52:45.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:45.045104  9915 server_base.cc:1179] Dumped server information to /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/ts-2/data/info.pb
21:52:45.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:45.055569  9331 external_mini_cluster.cc:1428] Started /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu as pid 9915
21:52:45.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:45.056196  9331 external_mini_cluster.cc:1442] Reading /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster7155462832654714522/ts-2/wal/instance
21:52:45.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:45.135198 10046 heartbeater.cc:344] Connected to a master server at 127.9.28.253:41061
21:52:45.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:45.135838 10046 heartbeater.cc:461] Registering TS with master...
21:52:45.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:45.136358 10045 heartbeater.cc:344] Connected to a master server at 127.9.28.252:40867
21:52:45.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:45.136739 10045 heartbeater.cc:461] Registering TS with master...
21:52:45.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:45.136878 10047 heartbeater.cc:344] Connected to a master server at 127.9.28.254:37687
21:52:45.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:45.137446 10047 heartbeater.cc:461] Registering TS with master...
21:52:45.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:45.137395 10046 heartbeater.cc:507] Master 127.9.28.253:41061 requested a full tablet report, sending...
21:52:45.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:45.137949 10045 heartbeater.cc:507] Master 127.9.28.252:40867 requested a full tablet report, sending...
21:52:45.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:45.138789 10047 heartbeater.cc:507] Master 127.9.28.254:37687 requested a full tablet report, sending...
21:52:45.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:45.141445  9451 ts_manager.cc:194] Registered new tserver with Master: 245563a49aa14536a93528a012b68226 (127.9.28.195:39395)
21:52:45.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:45.141126  9531 ts_manager.cc:194] Registered new tserver with Master: 245563a49aa14536a93528a012b68226 (127.9.28.195:39395)
21:52:45.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:45.142194  9376 ts_manager.cc:194] Registered new tserver with Master: 245563a49aa14536a93528a012b68226 (127.9.28.195:39395)
21:52:45.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:45.143755  9531 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.9.28.195:48599
21:52:45.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:45.150861  9331 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
21:52:45.163 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
21:52:45.163 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}

21:52:45.164 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
  masters {
    id {
      type: MASTER
      index: 0
    }
    bound_rpc_address {
      host: "127.9.28.254"
      port: 37687
    }
    bound_http_address {
      host: "127.9.28.254"
      port: 42375
    }
  }
  masters {
    id {
      type: MASTER
      index: 1
    }
    bound_rpc_address {
      host: "127.9.28.253"
      port: 41061
    }
    bound_http_address {
      host: "127.9.28.253"
      port: 33853
    }
  }
  masters {
    id {
      type: MASTER
      index: 2
    }
    bound_rpc_address {
      host: "127.9.28.252"
      port: 40867
    }
    bound_http_address {
      host: "127.9.28.252"
      port: 33487
    }
  }
}

21:52:45.165 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}

21:52:45.166 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
  tservers {
    id {
      type: TSERVER
      index: 0
    }
    bound_rpc_address {
      host: "127.9.28.193"
      port: 33195
    }
    bound_http_address {
      host: "127.9.28.193"
      port: 35443
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 1
    }
    bound_rpc_address {
      host: "127.9.28.194"
      port: 34061
    }
    bound_http_address {
      host: "127.9.28.194"
      port: 37873
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 2
    }
    bound_rpc_address {
      host: "127.9.28.195"
      port: 39395
    }
    bound_http_address {
      host: "127.9.28.195"
      port: 36253
    }
  }
}

21:52:45.166 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
21:52:45.167 [DEBUG - main] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2207099081988034621/krb5kdc/krb5cc
21:52:45.168 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 
21:52:45.168 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-tasktmwPRy/test-tmp
21:52:45.169 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
21:52:45.171 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-tasktmwPRy/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
21:52:45.173 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
  num_masters: 3
  num_tservers: 0
  enable_kerberos: true
  cluster_root: "/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466"
  extra_master_flags: "--webserver_require_spnego=false"
  extra_master_flags: "--authn_token_validity_seconds=10"
  extra_tserver_flags: "--webserver_require_spnego=false"
  hms_mode: NONE
  mini_kdc_options {
    ticket_lifetime: "10s"
    renew_lifetime: "20s"
  }
  principal: "kudu"
  mini_oidc_options {
  }
}

21:52:45.564 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:52:45.567 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}

21:52:45.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:45.571696 10056 mini_oidc.cc:150] Starting JWKS server
21:52:45.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:45.575493 10056 webserver.cc:480] Webserver started at http://127.0.0.1:42267/ using document root <none> and password file <none>
21:52:45.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:45.575994 10056 mini_oidc.cc:186] Starting OIDC Discovery server
21:52:45.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:45.578004 10056 webserver.cc:480] Webserver started at http://127.0.0.1:46081/ using document root <none> and password file <none>
21:52:45.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
21:52:45.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/krb5kdc/principal' for realm 'KRBTEST.COM',
21:52:45.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
21:52:45.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:52:45 dist-test-slave-q7qm krb5kdc[10067](info): setting up network...
21:52:45.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
21:52:45.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:52:45 dist-test-slave-q7qm krb5kdc[10067](info): set up 2 sockets
21:52:45.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:52:45 dist-test-slave-q7qm krb5kdc[10067](info): commencing operation
21:52:45.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
21:52:46.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:46.149482 10045 heartbeater.cc:499] Master 127.9.28.252:40867 was elected leader, sending a full tablet report...
21:52:47.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:47.980485 10056 mini_kdc.cc:121] Time spent starting KDC: real 2.344s	user 0.005s	sys 0.019s
21:52:47.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
21:52:48.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
21:52:48.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
21:52:48.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
21:52:48.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
21:52:48.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
21:52:48.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
21:52:48.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
21:52:48.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
21:52:48.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
21:52:48.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/krb5kdc/test-user.keytab.
21:52:48.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/krb5kdc/test-user.keytab.
21:52:48.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:52:48 dist-test-slave-q7qm krb5kdc[10067](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849968, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:52:48.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM: 
21:52:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-02T21:52:48Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
21:52:48.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-02T21:52:48Z Disabled control of system clock
21:52:48.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.9.210.62@KRBTEST.COM; defaulting to no policy
21:52:48.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:52:48.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.9.210.62@KRBTEST.COM" created.
21:52:48.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:52:48.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.9.210.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/krb5kdc/kudu.keytab.
21:52:48.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.9.210.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/krb5kdc/kudu.keytab.
21:52:48.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.9.210.62@KRBTEST.COM; defaulting to no policy
21:52:48.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:52:48.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.9.210.62@KRBTEST.COM" created.
21:52:48.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:52:48.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.9.210.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/krb5kdc/kudu.keytab.
21:52:48.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.9.210.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/krb5kdc/kudu.keytab.
21:52:48.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:48.260587 10056 external_mini_cluster.cc:1366] Running /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu
21:52:48.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu
21:52:48.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-0/wal
21:52:48.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-0/data
21:52:48.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:52:48.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:52:48.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:52:48.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:52:48.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:52:48.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:52:48.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-0/logs
21:52:48.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-0/data/info.pb
21:52:48.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:52:48.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:52:48.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:52:48.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:52:48.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:52:48.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:52:48.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:52:48.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:52:48.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:52:48.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:52:48.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:52:48.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:52:48.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.210.62:42721
21:52:48.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.210.62
21:52:48.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:52:48.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/krb5kdc/kudu.keytab
21:52:48.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.210.62
21:52:48.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:52:48.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:52:48.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:52:48.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:52:48.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.210.20:37705
21:52:48.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:52:48.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:52:48.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:52:48.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:52:48.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.210.62:42721,127.9.210.61:38683,127.9.210.60:36675
21:52:48.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:52:48.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42267/jwks.json
21:52:48.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
21:52:48.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:52:48.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:48.585003 10084 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:52:48.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:48.585626 10084 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:52:48.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:48.586524 10084 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:52:48.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:48.623077 10084 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:52:48.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:48.623449 10084 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:52:48.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:48.624004 10084 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:52:48.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:48.624334 10084 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:52:48.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:48.624765 10084 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:52:48.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:48.625113 10084 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:52:48.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:48.625337 10084 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42267/jwks.json
21:52:48.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:48.669139 10084 master_runner.cc:386] Master server non-default flags:
21:52:48.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:52:48.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.210.20:37705
21:52:48.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:52:48.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:52:48.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-0/data
21:52:48.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-0/wal
21:52:48.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:52:48.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:52:48.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.210.62:42721,127.9.210.61:38683,127.9.210.60:36675
21:52:48.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/krb5kdc/kudu.keytab
21:52:48.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.210.62
21:52:48.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:52:48.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:52:48.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:52:48.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.210.62:42721
21:52:48.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:52:48.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:52:48.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:52:48.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42267/jwks.json
21:52:48.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:52:48.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:52:48.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:52:48.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:52:48.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:52:48.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-0/data/info.pb
21:52:48.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.210.62
21:52:48.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:52:48.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:52:48.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:52:48.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:52:48.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:52:48.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:52:48.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-0/logs
21:52:48.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:52:48.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:52:48.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:52:48.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:52:48.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:52:48.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6ab3f249351a4ee5d73eb8bda3585c4dcd85d070
21:52:48.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:52:48.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 21:30:03 UTC on 5fd53c4cbb9d
21:52:48.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7869
21:52:48.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:52:48.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:48.670799 10084 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:52:48.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:48.672111 10084 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:52:48.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:48.684587 10090 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:52:50.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:50.087926 10089 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 10084
21:52:50.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:50.108052 10089 kernel_stack_watchdog.cc:198] Thread 10084 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 399ms:
21:52:50.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
21:52:50.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
21:52:50.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
21:52:50.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
21:52:50.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:48.684901 10091 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:52:50.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:50.109102 10084 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.423s	user 0.000s	sys 0.002s
21:52:50.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:50.109547 10084 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.424s	user 0.000s	sys 0.002s
21:52:50.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:50.111863 10093 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:52:50.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:50.117527 10084 server_base.cc:1047] running on GCE node
21:52:50.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:50.118965 10084 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:52:50.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:50.121919 10084 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:52:50.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:50.123391 10084 hybrid_clock.cc:648] HybridClock initialized: now 1756849970123312 us; error 45 us; skew 500 ppm
21:52:50.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:50.124442 10084 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:52:50.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:52:50 dist-test-slave-q7qm krb5kdc[10067](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849970, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.210.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:52:50.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:50.131307 10084 init.cc:377] Logged in from keytab as kudu/127.9.210.62@KRBTEST.COM (short username kudu)
21:52:50.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:50.136847 10084 webserver.cc:480] Webserver started at http://127.9.210.62:46869/ using document root <none> and password file <none>
21:52:50.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:50.137791 10084 fs_manager.cc:362] Metadata directory not provided
21:52:50.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:50.137993 10084 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:52:50.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:50.138392 10084 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:52:50.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:50.143739 10084 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-0/data/instance:
21:52:50.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6b28dbbfdb35431095e95733b2e72246"
21:52:50.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:52:50 on dist-test-slave-q7qm"
21:52:50.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:50.145447 10084 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-0/wal/instance:
21:52:50.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6b28dbbfdb35431095e95733b2e72246"
21:52:50.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:52:50 on dist-test-slave-q7qm"
21:52:50.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:50.153576 10084 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.009s	sys 0.000s
21:52:50.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:50.160259 10103 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:52:50.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:50.161602 10084 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.004s	sys 0.000s
21:52:50.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:50.162074 10084 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-0/data,/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-0/wal
21:52:50.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6b28dbbfdb35431095e95733b2e72246"
21:52:50.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:52:50 on dist-test-slave-q7qm"
21:52:50.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:50.162447 10084 fs_report.cc:389] FS layout report
21:52:50.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:52:50.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-0/wal
21:52:50.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-0/wal
21:52:50.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-0/data/data
21:52:50.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:52:50.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:52:50.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:52:50.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:52:50.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:52:50.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:52:50.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:52:50.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:52:50.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:52:50.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:52:50.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:52:50.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:50.227916 10084 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:52:50.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:50.239024 10084 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:52:50.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:50.239578 10084 kserver.cc:163] Server-wide thread pool size limit: 3276
21:52:50.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:50.326342 10084 rpc_server.cc:307] RPC server started. Bound to: 127.9.210.62:42721
21:52:50.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:50.326427 10155 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.210.62:42721 every 8 connection(s)
21:52:50.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:50.329159 10084 server_base.cc:1179] Dumped server information to /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-0/data/info.pb
21:52:50.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:50.334692 10156 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:52:50.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:50.336181 10056 external_mini_cluster.cc:1428] Started /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu as pid 10084
21:52:50.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:50.336748 10056 external_mini_cluster.cc:1442] Reading /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-0/wal/instance
21:52:50.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:50.351133 10156 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.210.62" port: 42721 } has no permanent_uuid. Determining permanent_uuid...
21:52:50.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.9.210.61@KRBTEST.COM; defaulting to no policy
21:52:50.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:52:50.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.9.210.61@KRBTEST.COM" created.
21:52:50.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:52:50 dist-test-slave-q7qm krb5kdc[10067](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849970, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.210.62@KRBTEST.COM for kudu/127.9.210.62@KRBTEST.COM
21:52:50.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:52:50.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.9.210.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/krb5kdc/kudu.keytab.
21:52:50.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.9.210.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/krb5kdc/kudu.keytab.
21:52:50.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:50.396644 10156 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.210.61" port: 38683 } has no permanent_uuid. Determining permanent_uuid...
21:52:50.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:50.398948 10107 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.210.61:38683: connect: Connection refused (error 111)
21:52:50.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.9.210.61@KRBTEST.COM; defaulting to no policy
21:52:50.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:52:50.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.9.210.61@KRBTEST.COM" created.
21:52:50.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:50.403426 10156 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.9.210.61:38683: Network error: Client connection negotiation failed: client connection to 127.9.210.61:38683: connect: Connection refused (error 111)
21:52:50.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:52:50.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.9.210.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/krb5kdc/kudu.keytab.
21:52:50.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.9.210.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/krb5kdc/kudu.keytab.
21:52:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:50.429752 10056 external_mini_cluster.cc:1366] Running /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu
21:52:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu
21:52:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-1/wal
21:52:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-1/data
21:52:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:52:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:52:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:52:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:52:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:52:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:52:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-1/logs
21:52:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-1/data/info.pb
21:52:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:52:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:52:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:52:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:52:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:52:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:52:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:52:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:52:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:52:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:52:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:52:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:52:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.210.61:38683
21:52:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.210.61
21:52:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:52:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/krb5kdc/kudu.keytab
21:52:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.210.61
21:52:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:52:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:52:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:52:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:52:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.210.20:37705
21:52:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:52:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:52:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:52:50.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:52:50.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.210.62:42721,127.9.210.61:38683,127.9.210.60:36675
21:52:50.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:52:50.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42267/jwks.json
21:52:50.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
21:52:50.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:52:50.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:50.452972 10156 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.210.61" port: 38683 } attempt: 1
21:52:50.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:50.457024 10156 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.9.210.61:38683: Network error: Client connection negotiation failed: client connection to 127.9.210.61:38683: connect: Connection refused (error 111)
21:52:50.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:50.525449 10156 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.210.61" port: 38683 } attempt: 2
21:52:50.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:50.530830 10156 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.9.210.61:38683: Network error: Client connection negotiation failed: client connection to 127.9.210.61:38683: connect: Connection refused (error 111)
21:52:50.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:50.622416 10156 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.210.61" port: 38683 } attempt: 3
21:52:50.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:50.626717 10156 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.9.210.61:38683: Network error: Client connection negotiation failed: client connection to 127.9.210.61:38683: connect: Connection refused (error 111)
21:52:50.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:50.770150 10156 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.210.61" port: 38683 } attempt: 4
21:52:50.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:50.772053 10165 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:52:50.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:50.772725 10165 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:52:50.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:50.773880 10165 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:52:50.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:50.774482 10156 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.9.210.61:38683: Network error: Client connection negotiation failed: client connection to 127.9.210.61:38683: connect: Connection refused (error 111)
21:52:50.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:50.808485 10165 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:52:50.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:50.808771 10165 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:52:50.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:50.809145 10165 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:52:50.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:50.809439 10165 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:52:50.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:50.809716 10165 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:52:50.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:50.809984 10165 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:52:50.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:50.810190 10165 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42267/jwks.json
21:52:50.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:50.875083 10165 master_runner.cc:386] Master server non-default flags:
21:52:50.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:52:50.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.210.20:37705
21:52:50.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:52:50.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:52:50.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-1/data
21:52:50.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-1/wal
21:52:50.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:52:50.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:52:50.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.210.62:42721,127.9.210.61:38683,127.9.210.60:36675
21:52:50.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/krb5kdc/kudu.keytab
21:52:50.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.210.61
21:52:50.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:52:50.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:52:50.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:52:50.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.210.61:38683
21:52:50.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:52:50.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:52:50.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:52:50.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42267/jwks.json
21:52:50.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:52:50.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:52:50.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:52:50.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:52:50.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:52:50.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-1/data/info.pb
21:52:50.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.210.61
21:52:50.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:52:50.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:52:50.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:52:50.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:52:50.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:52:50.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:52:50.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-1/logs
21:52:50.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:52:50.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:52:50.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:52:50.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:52:50.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:52:50.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6ab3f249351a4ee5d73eb8bda3585c4dcd85d070
21:52:50.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:52:50.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 21:30:03 UTC on 5fd53c4cbb9d
21:52:50.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7869
21:52:50.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:52:50.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:50.876828 10165 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:52:50.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:50.878182 10165 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:52:50.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:50.891589 10171 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:52:51.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:51.073935 10156 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.210.61" port: 38683 } attempt: 5
21:52:51.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:51.080904 10156 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.9.210.61:38683: Network error: Client connection negotiation failed: client connection to 127.9.210.61:38683: connect: Connection refused (error 111)
21:52:51.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:51.628580 10156 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.210.61" port: 38683 } attempt: 6
21:52:51.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:51.657454 10156 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.9.210.61:38683: Network error: Client connection negotiation failed: client connection to 127.9.210.61:38683: connect: Connection refused (error 111)
21:52:52.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:50.893790 10172 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:52:52.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:50.897795 10174 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:52:52.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:52.237229 10173 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1336 milliseconds
21:52:52.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:52.237371 10165 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:52:52.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:52.238900 10165 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:52:52.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:52.242137 10165 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:52:52.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:52.243707 10165 hybrid_clock.cc:648] HybridClock initialized: now 1756849972243642 us; error 96 us; skew 500 ppm
21:52:52.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:52.244830 10165 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:52:52.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:52:52 dist-test-slave-q7qm krb5kdc[10067](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849972, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.210.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:52:52.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:52.253140 10165 init.cc:377] Logged in from keytab as kudu/127.9.210.61@KRBTEST.COM (short username kudu)
21:52:52.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:52.259126 10165 webserver.cc:480] Webserver started at http://127.9.210.61:38571/ using document root <none> and password file <none>
21:52:52.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:52.260426 10165 fs_manager.cc:362] Metadata directory not provided
21:52:52.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:52.260715 10165 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:52:52.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:52.261300 10165 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:52:52.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:52.268944 10165 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-1/data/instance:
21:52:52.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "29c607b525464ef1bf7fc15526b3a056"
21:52:52.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:52:52 on dist-test-slave-q7qm"
21:52:52.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:52.270644 10165 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-1/wal/instance:
21:52:52.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "29c607b525464ef1bf7fc15526b3a056"
21:52:52.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:52:52 on dist-test-slave-q7qm"
21:52:52.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:52.279073 10165 fs_manager.cc:696] Time spent creating directory manager: real 0.008s	user 0.002s	sys 0.004s
21:52:52.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:52.286329 10186 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:52:52.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:52.287889 10165 fs_manager.cc:730] Time spent opening block manager: real 0.005s	user 0.006s	sys 0.000s
21:52:52.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:52.288678 10165 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-1/data,/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-1/wal
21:52:52.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "29c607b525464ef1bf7fc15526b3a056"
21:52:52.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:52:52 on dist-test-slave-q7qm"
21:52:52.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:52.289551 10165 fs_report.cc:389] FS layout report
21:52:52.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:52:52.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-1/wal
21:52:52.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-1/wal
21:52:52.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-1/data/data
21:52:52.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:52:52.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:52:52.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:52:52.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:52:52.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:52:52.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:52:52.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:52:52.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:52:52.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:52:52.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:52:52.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:52:52.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:52.360440 10165 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:52:52.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:52.366353 10165 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:52:52.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:52.367089 10165 kserver.cc:163] Server-wide thread pool size limit: 3276
21:52:52.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:52.452957 10165 rpc_server.cc:307] RPC server started. Bound to: 127.9.210.61:38683
21:52:52.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:52.453074 10237 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.210.61:38683 every 8 connection(s)
21:52:52.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:52.455885 10165 server_base.cc:1179] Dumped server information to /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-1/data/info.pb
21:52:52.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:52.459690 10056 external_mini_cluster.cc:1428] Started /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu as pid 10165
21:52:52.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:52.460206 10056 external_mini_cluster.cc:1442] Reading /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-1/wal/instance
21:52:52.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:52.462661 10238 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:52:52.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.9.210.60@KRBTEST.COM; defaulting to no policy
21:52:52.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:52.478339 10238 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.210.62" port: 42721 } has no permanent_uuid. Determining permanent_uuid...
21:52:52.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:52:52.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.9.210.60@KRBTEST.COM" created.
21:52:52.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:52:52 dist-test-slave-q7qm krb5kdc[10067](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849972, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.210.61@KRBTEST.COM for kudu/127.9.210.62@KRBTEST.COM
21:52:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:52:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.9.210.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/krb5kdc/kudu.keytab.
21:52:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.9.210.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/krb5kdc/kudu.keytab.
21:52:52.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:52.530002 10238 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.210.61" port: 38683 } has no permanent_uuid. Determining permanent_uuid...
21:52:52.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.9.210.60@KRBTEST.COM; defaulting to no policy
21:52:52.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:52:52.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.9.210.60@KRBTEST.COM" created.
21:52:52.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:52:52 dist-test-slave-q7qm krb5kdc[10067](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849972, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.210.61@KRBTEST.COM for kudu/127.9.210.61@KRBTEST.COM
21:52:52.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:52.563045 10238 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.210.60" port: 36675 } has no permanent_uuid. Determining permanent_uuid...
21:52:52.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:52.565711 10187 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.210.60:36675: connect: Connection refused (error 111)
21:52:52.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:52.570722 10238 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.9.210.60:36675: Network error: Client connection negotiation failed: client connection to 127.9.210.60:36675: connect: Connection refused (error 111)
21:52:52.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:52:52.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.9.210.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/krb5kdc/kudu.keytab.
21:52:52.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.9.210.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/krb5kdc/kudu.keytab.
21:52:52.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:52.574463 10056 external_mini_cluster.cc:1366] Running /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu
21:52:52.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu
21:52:52.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-2/wal
21:52:52.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-2/data
21:52:52.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:52:52.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:52:52.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:52:52.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:52:52.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:52:52.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:52:52.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-2/logs
21:52:52.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-2/data/info.pb
21:52:52.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:52:52.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:52:52.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:52:52.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:52:52.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:52:52.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:52:52.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:52:52.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:52:52.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:52:52.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:52:52.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:52:52.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:52:52.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.210.60:36675
21:52:52.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.210.60
21:52:52.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:52:52.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/krb5kdc/kudu.keytab
21:52:52.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.210.60
21:52:52.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:52:52.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:52:52.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:52:52.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:52:52.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.210.20:37705
21:52:52.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:52:52.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:52:52.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:52:52.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:52:52.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.210.62:42721,127.9.210.61:38683,127.9.210.60:36675
21:52:52.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:52:52.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42267/jwks.json
21:52:52.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
21:52:52.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:52:52.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:52.620395 10238 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.210.60" port: 36675 } attempt: 1
21:52:52.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:52.625813 10238 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.9.210.60:36675: Network error: Client connection negotiation failed: client connection to 127.9.210.60:36675: connect: Connection refused (error 111)
21:52:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:52.694262 10238 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.210.60" port: 36675 } attempt: 2
21:52:52.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:52.698369 10238 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.9.210.60:36675: Network error: Client connection negotiation failed: client connection to 127.9.210.60:36675: connect: Connection refused (error 111)
21:52:52.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:52.718210 10156 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.210.61" port: 38683 } attempt: 7
21:52:52.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:52:52 dist-test-slave-q7qm krb5kdc[10067](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849970, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.210.62@KRBTEST.COM for kudu/127.9.210.61@KRBTEST.COM
21:52:52.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:52.751523 10156 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.210.60" port: 36675 } has no permanent_uuid. Determining permanent_uuid...
21:52:52.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:52.758885 10156 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.9.210.60:36675: Network error: Client connection negotiation failed: client connection to 127.9.210.60:36675: connect: Connection refused (error 111)
21:52:52.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:52.789906 10238 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.210.60" port: 36675 } attempt: 3
21:52:52.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:52.794761 10238 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.9.210.60:36675: Network error: Client connection negotiation failed: client connection to 127.9.210.60:36675: connect: Connection refused (error 111)
21:52:52.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:52.817315 10156 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.210.60" port: 36675 } attempt: 1
21:52:52.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:52.822304 10156 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.9.210.60:36675: Network error: Client connection negotiation failed: client connection to 127.9.210.60:36675: connect: Connection refused (error 111)
21:52:52.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:52.903827 10156 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.210.60" port: 36675 } attempt: 2
21:52:52.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:52.908571 10156 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.9.210.60:36675: Network error: Client connection negotiation failed: client connection to 127.9.210.60:36675: connect: Connection refused (error 111)
21:52:52.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:52.938244 10238 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.210.60" port: 36675 } attempt: 4
21:52:52.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:52.942849 10238 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.9.210.60:36675: Network error: Client connection negotiation failed: client connection to 127.9.210.60:36675: connect: Connection refused (error 111)
21:52:52.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:52.961161 10248 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:52:52.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:52.961759 10248 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:52:52.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:52.962401 10248 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:52:52.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:52.994019 10156 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.210.60" port: 36675 } attempt: 3
21:52:52.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:52.996229 10248 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:52:52.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:52.996656 10248 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:52:52.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:52.997287 10248 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:52:52.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:52.997663 10248 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:52:52.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:52.997889 10248 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:52:52.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:52.998219 10248 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:52:52.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:52.998653 10248 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42267/jwks.json
21:52:52.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:52.999291 10156 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.9.210.60:36675: Network error: Client connection negotiation failed: client connection to 127.9.210.60:36675: connect: Connection refused (error 111)
21:52:53.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:53.037290 10248 master_runner.cc:386] Master server non-default flags:
21:52:53.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:52:53.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.210.20:37705
21:52:53.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:52:53.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:52:53.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-2/data
21:52:53.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-2/wal
21:52:53.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:52:53.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:52:53.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.210.62:42721,127.9.210.61:38683,127.9.210.60:36675
21:52:53.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/krb5kdc/kudu.keytab
21:52:53.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.210.60
21:52:53.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:52:53.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:52:53.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:52:53.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.210.60:36675
21:52:53.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:52:53.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:52:53.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:52:53.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42267/jwks.json
21:52:53.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:52:53.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:52:53.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:52:53.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:52:53.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:52:53.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-2/data/info.pb
21:52:53.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.210.60
21:52:53.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:52:53.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:52:53.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:52:53.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:52:53.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:52:53.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:52:53.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-2/logs
21:52:53.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:52:53.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:52:53.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:52:53.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:52:53.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:52:53.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6ab3f249351a4ee5d73eb8bda3585c4dcd85d070
21:52:53.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:52:53.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 21:30:03 UTC on 5fd53c4cbb9d
21:52:53.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7869
21:52:53.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:52:53.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:53.038959 10248 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:52:53.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:53.040374 10248 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:52:53.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:53.054033 10256 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:52:53.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:53.139791 10156 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.210.60" port: 36675 } attempt: 4
21:52:53.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:53.147838 10156 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.9.210.60:36675: Network error: Client connection negotiation failed: client connection to 127.9.210.60:36675: connect: Connection refused (error 111)
21:52:53.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:53.242372 10238 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.210.60" port: 36675 } attempt: 5
21:52:53.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:53.250236 10238 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.9.210.60:36675: Network error: Client connection negotiation failed: client connection to 127.9.210.60:36675: connect: Connection refused (error 111)
21:52:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:53.432827 10156 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.210.60" port: 36675 } attempt: 5
21:52:53.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:53.446282 10156 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.9.210.60:36675: Network error: Client connection negotiation failed: client connection to 127.9.210.60:36675: connect: Connection refused (error 111)
21:52:53.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:53.797807 10238 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.210.60" port: 36675 } attempt: 6
21:52:53.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:53.814682 10238 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.9.210.60:36675: Network error: Client connection negotiation failed: client connection to 127.9.210.60:36675: connect: Connection refused (error 111)
21:52:54.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:53.999053 10156 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.210.60" port: 36675 } attempt: 6
21:52:54.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:54.033515 10156 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.9.210.60:36675: Network error: Client connection negotiation failed: client connection to 127.9.210.60:36675: connect: Connection refused (error 111)
21:52:54.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:54.875288 10238 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.210.60" port: 36675 } attempt: 7
21:52:54.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:54.459658 10255 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 10248
21:52:54.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:53.056623 10257 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:52:54.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:54.880326 10248 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.825s	user 0.575s	sys 1.145s
21:52:54.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:54.880722 10248 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.825s	user 0.575s	sys 1.145s
21:52:54.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:54.883347 10259 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:52:54.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:54.885726 10258 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1830 milliseconds
21:52:54.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:54.885850 10248 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:52:54.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:54.887773 10248 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:52:54.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:54.890650 10238 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.9.210.60:36675: Network error: Client connection negotiation failed: client connection to 127.9.210.60:36675: connect: Connection refused (error 111)
21:52:54.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:54.890841 10248 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:52:54.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:54.892261 10248 hybrid_clock.cc:648] HybridClock initialized: now 1756849974892212 us; error 53 us; skew 500 ppm
21:52:54.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:54.893189 10248 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:52:54.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:52:54 dist-test-slave-q7qm krb5kdc[10067](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849974, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.210.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:52:54.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:54.900107 10248 init.cc:377] Logged in from keytab as kudu/127.9.210.60@KRBTEST.COM (short username kudu)
21:52:54.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:54.905436 10248 webserver.cc:480] Webserver started at http://127.9.210.60:33869/ using document root <none> and password file <none>
21:52:54.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:54.906436 10248 fs_manager.cc:362] Metadata directory not provided
21:52:54.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:54.906683 10248 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:52:54.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:54.907176 10248 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:52:54.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:54.912603 10248 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-2/data/instance:
21:52:54.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6a711b14746d4eb6baba23e55bdaac69"
21:52:54.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:52:54 on dist-test-slave-q7qm"
21:52:54.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:54.913933 10248 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-2/wal/instance:
21:52:54.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6a711b14746d4eb6baba23e55bdaac69"
21:52:54.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:52:54 on dist-test-slave-q7qm"
21:52:54.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:54.922220 10248 fs_manager.cc:696] Time spent creating directory manager: real 0.008s	user 0.010s	sys 0.000s
21:52:54.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:54.928514 10275 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:52:54.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:54.929719 10248 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.002s
21:52:54.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:54.930101 10248 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-2/data,/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-2/wal
21:52:54.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6a711b14746d4eb6baba23e55bdaac69"
21:52:54.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:52:54 on dist-test-slave-q7qm"
21:52:54.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:54.930526 10248 fs_report.cc:389] FS layout report
21:52:54.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:52:54.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-2/wal
21:52:54.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-2/wal
21:52:54.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-2/data/data
21:52:54.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:52:54.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:52:54.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:52:54.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:52:54.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:52:54.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:52:54.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:52:54.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:52:54.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:52:54.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:52:54.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:52:54.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:54.983995 10248 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:52:54.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:54.991652 10248 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:52:54.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:54.992086 10248 kserver.cc:163] Server-wide thread pool size limit: 3276
21:52:55.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:55.067167 10156 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.210.60" port: 36675 } attempt: 7
21:52:55.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:55.074196 10156 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.9.210.60:36675: Network error: Client connection negotiation failed: client connection to 127.9.210.60:36675: connect: Connection refused (error 111)
21:52:55.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:55.078902 10248 rpc_server.cc:307] RPC server started. Bound to: 127.9.210.60:36675
21:52:55.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:55.079020 10328 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.210.60:36675 every 8 connection(s)
21:52:55.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:55.081655 10248 server_base.cc:1179] Dumped server information to /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-2/data/info.pb
21:52:55.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:55.083995 10056 external_mini_cluster.cc:1428] Started /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu as pid 10248
21:52:55.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:55.084535 10056 external_mini_cluster.cc:1442] Reading /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-2/wal/instance
21:52:55.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:55.089265 10329 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:52:55.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:55.113346 10329 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.210.62" port: 42721 } has no permanent_uuid. Determining permanent_uuid...
21:52:55.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:52:55 dist-test-slave-q7qm krb5kdc[10067](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849968, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.9.210.60@KRBTEST.COM
21:52:55.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:52:55 dist-test-slave-q7qm krb5kdc[10067](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849974, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.210.60@KRBTEST.COM for kudu/127.9.210.62@KRBTEST.COM
21:52:55.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:52:55 dist-test-slave-q7qm krb5kdc[10067](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849968, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.9.210.61@KRBTEST.COM
21:52:55.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:55.163918 10329 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.210.61" port: 38683 } has no permanent_uuid. Determining permanent_uuid...
21:52:55.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:52:55 dist-test-slave-q7qm krb5kdc[10067](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849974, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.210.60@KRBTEST.COM for kudu/127.9.210.61@KRBTEST.COM
21:52:55.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:52:55 dist-test-slave-q7qm krb5kdc[10067](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849968, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.9.210.62@KRBTEST.COM
21:52:55.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:55.194801 10329 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.210.60" port: 36675 } has no permanent_uuid. Determining permanent_uuid...
21:52:55.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:52:55 dist-test-slave-q7qm krb5kdc[10067](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849974, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.210.60@KRBTEST.COM for kudu/127.9.210.60@KRBTEST.COM
21:52:55.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:55.218511 10056 external_mini_cluster.cc:949] 0 TS(s) registered with all masters
21:52:55.226 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:52:55.227 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}

21:52:55.229 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
  env_vars {
    key: "KRB5CCNAME"
    value: "/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/krb5kdc/krb5cc"
  }
  env_vars {
    key: "KRB5_CONFIG"
    value: "/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/krb5kdc/krb5.conf"
  }
  env_vars {
    key: "KRB5_KDC_PROFILE"
    value: "/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/krb5kdc/kdc.conf"
  }
  env_vars {
    key: "KUDU_ENABLE_KRB5_REALM_FIX"
    value: "yes"
  }
}

21:52:55.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:55.230643 10329 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69: Bootstrap starting.
21:52:55.231 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}

21:52:55.232 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
  masters {
    id {
      type: MASTER
      index: 0
    }
    bound_rpc_address {
      host: "127.9.210.62"
      port: 42721
    }
    bound_http_address {
      host: "127.9.210.62"
      port: 46869
    }
  }
  masters {
    id {
      type: MASTER
      index: 1
    }
    bound_rpc_address {
      host: "127.9.210.61"
      port: 38683
    }
    bound_http_address {
      host: "127.9.210.61"
      port: 38571
    }
  }
  masters {
    id {
      type: MASTER
      index: 2
    }
    bound_rpc_address {
      host: "127.9.210.60"
      port: 36675
    }
    bound_http_address {
      host: "127.9.210.60"
      port: 33869
    }
  }
}

21:52:55.233 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}

21:52:55.233 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:52:55.233 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
21:52:55.233 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
  username: "test-admin"
}

21:52:55.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:55.237751 10329 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69: Neither blocks nor log segments found. Creating new log.
21:52:55.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:55.240219 10329 log.cc:826] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69: Log is configured to *not* fsync() on all Append() calls
21:52:55.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:55.247685 10329 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69: No bootstrap required, opened a new log
21:52:55.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:52:55 dist-test-slave-q7qm krb5kdc[10067](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849975, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:52:55.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM: 
21:52:55.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:55.275748 10329 raft_consensus.cc:357] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b28dbbfdb35431095e95733b2e72246" member_type: VOTER last_known_addr { host: "127.9.210.62" port: 42721 } } peers { permanent_uuid: "29c607b525464ef1bf7fc15526b3a056" member_type: VOTER last_known_addr { host: "127.9.210.61" port: 38683 } } peers { permanent_uuid: "6a711b14746d4eb6baba23e55bdaac69" member_type: VOTER last_known_addr { host: "127.9.210.60" port: 36675 } }
21:52:55.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:55.276654 10329 raft_consensus.cc:383] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:52:55.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:55.276980 10329 raft_consensus.cc:738] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a711b14746d4eb6baba23e55bdaac69, State: Initialized, Role: FOLLOWER
21:52:55.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:55.277904 10329 consensus_queue.cc:260] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b28dbbfdb35431095e95733b2e72246" member_type: VOTER last_known_addr { host: "127.9.210.62" port: 42721 } } peers { permanent_uuid: "29c607b525464ef1bf7fc15526b3a056" member_type: VOTER last_known_addr { host: "127.9.210.61" port: 38683 } } peers { permanent_uuid: "6a711b14746d4eb6baba23e55bdaac69" member_type: VOTER last_known_addr { host: "127.9.210.60" port: 36675 } }
21:52:55.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:55.280754 10340 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b28dbbfdb35431095e95733b2e72246" member_type: VOTER last_known_addr { host: "127.9.210.62" port: 42721 } } peers { permanent_uuid: "29c607b525464ef1bf7fc15526b3a056" member_type: VOTER last_known_addr { host: "127.9.210.61" port: 38683 } } peers { permanent_uuid: "6a711b14746d4eb6baba23e55bdaac69" member_type: VOTER last_known_addr { host: "127.9.210.60" port: 36675 } } }
21:52:55.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:55.281494 10340 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [sys.catalog]: This master's current role is: FOLLOWER
21:52:55.283 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:52:55.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:55.283803 10329 sys_catalog.cc:564] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [sys.catalog]: configured and running, proceeding with master startup.
21:52:55.284 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/krb5kdc/krb5cc
21:52:55.287 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
21:52:55.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:55.313172 10351 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:52:55.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:55.313733 10351 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:52:55.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:55.309976 10248 master_runner.cc:186] Error getting master registration for 127.9.210.62:42721: OK, instance_id { permanent_uuid: "6b28dbbfdb35431095e95733b2e72246" instance_seqno: 1756849970305081 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:52:55.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:55.341563 10248 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:52:55.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:55.497707 10281 tablet.cc:2378] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:52:55.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:55.503290 10340 raft_consensus.cc:491] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:52:55.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:55.504086 10340 raft_consensus.cc:513] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b28dbbfdb35431095e95733b2e72246" member_type: VOTER last_known_addr { host: "127.9.210.62" port: 42721 } } peers { permanent_uuid: "29c607b525464ef1bf7fc15526b3a056" member_type: VOTER last_known_addr { host: "127.9.210.61" port: 38683 } } peers { permanent_uuid: "6a711b14746d4eb6baba23e55bdaac69" member_type: VOTER last_known_addr { host: "127.9.210.60" port: 36675 } }
21:52:55.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:55.508785 10340 leader_election.cc:290] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6b28dbbfdb35431095e95733b2e72246 (127.9.210.62:42721), 29c607b525464ef1bf7fc15526b3a056 (127.9.210.61:38683)
21:52:56.289 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.9.210.61:38683
21:52:56.291 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: MASTER
    index: 1
  }
}

21:52:56.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:56.291762 10056 external_mini_cluster.cc:1658] Killing /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu with pid 10165
21:52:56.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:56.315618 10279 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv got EOF from 127.9.210.61:38683 (error 108)
21:52:56.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:56.318790 10351 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:52:56.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:56.319286 10351 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:52:56.324 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:52:56.324 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.9.210.62:42721
21:52:56.324 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: MASTER
    index: 0
  }
}

21:52:56.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:56.330060 10056 external_mini_cluster.cc:1658] Killing /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu with pid 10084
21:52:56.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:56.333962 10279 leader_election.cc:336] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 29c607b525464ef1bf7fc15526b3a056 (127.9.210.61:38683): Network error: Client connection negotiation failed: client connection to 127.9.210.61:38683: connect: Connection refused (error 111)
21:52:56.357 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:52:56.357 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.9.210.60:36675
21:52:56.357 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: MASTER
    index: 2
  }
}

21:52:56.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:56.357896 10056 external_mini_cluster.cc:1658] Killing /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu with pid 10248
21:52:56.386 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:52:56.387 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.9.210.61:38683
21:52:56.387 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: MASTER
    index: 1
  }
}

21:52:56.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:56.389221 10056 external_mini_cluster.cc:1366] Running /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu
21:52:56.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu
21:52:56.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-1/wal
21:52:56.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-1/data
21:52:56.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:52:56.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:52:56.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:52:56.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:52:56.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:52:56.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:52:56.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-1/logs
21:52:56.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-1/data/info.pb
21:52:56.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:52:56.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:52:56.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:52:56.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:52:56.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:52:56.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:52:56.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:52:56.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:52:56.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:52:56.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:52:56.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:52:56.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:52:56.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.210.61:38683
21:52:56.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.210.61
21:52:56.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38571
21:52:56.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/krb5kdc/kudu.keytab
21:52:56.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.210.61
21:52:56.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:52:56.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:52:56.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:52:56.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:52:56.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.210.20:37705
21:52:56.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:52:56.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:52:56.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:52:56.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:52:56.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.210.62:42721,127.9.210.61:38683,127.9.210.60:36675
21:52:56.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:52:56.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42267/jwks.json
21:52:56.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
21:52:56.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:52:56.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:56.759202 10357 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:52:56.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:56.759940 10357 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:52:56.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:56.760673 10357 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:52:56.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:56.795300 10357 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:52:56.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:56.795775 10357 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:52:56.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:56.796136 10357 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:52:56.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:56.796361 10357 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:52:56.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:56.796648 10357 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:52:56.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:56.796867 10357 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:52:56.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:56.797015 10357 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42267/jwks.json
21:52:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:56.840725 10357 master_runner.cc:386] Master server non-default flags:
21:52:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:52:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.210.20:37705
21:52:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:52:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:52:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-1/data
21:52:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-1/wal
21:52:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:52:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:52:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.210.62:42721,127.9.210.61:38683,127.9.210.60:36675
21:52:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/krb5kdc/kudu.keytab
21:52:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.210.61
21:52:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:52:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:52:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:52:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.210.61:38683
21:52:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:52:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:52:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:52:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42267/jwks.json
21:52:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:52:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:52:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:52:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:52:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:52:56.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-1/data/info.pb
21:52:56.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.210.61
21:52:56.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38571
21:52:56.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:52:56.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:52:56.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:52:56.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:52:56.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:52:56.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-1/logs
21:52:56.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:52:56.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:52:56.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:52:56.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:52:56.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:52:56.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6ab3f249351a4ee5d73eb8bda3585c4dcd85d070
21:52:56.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:52:56.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 21:30:03 UTC on 5fd53c4cbb9d
21:52:56.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7869
21:52:56.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:52:56.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:56.843034 10357 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:52:56.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:56.844544 10357 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:52:56.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:56.858150 10363 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:52:58.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:56.860770 10364 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:52:58.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:56.866400 10366 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:52:58.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:58.276258 10365 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
21:52:58.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:58.276575 10357 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:52:58.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:58.282783 10357 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:52:58.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:58.285641 10357 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:52:58.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:58.287110 10357 hybrid_clock.cc:648] HybridClock initialized: now 1756849978287061 us; error 53 us; skew 500 ppm
21:52:58.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:58.288077 10357 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:52:58.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:52:58 dist-test-slave-q7qm krb5kdc[10067](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849978, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.210.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:52:58.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:58.295281 10357 init.cc:377] Logged in from keytab as kudu/127.9.210.61@KRBTEST.COM (short username kudu)
21:52:58.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:58.300592 10357 webserver.cc:480] Webserver started at http://127.9.210.61:38571/ using document root <none> and password file <none>
21:52:58.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:58.301566 10357 fs_manager.cc:362] Metadata directory not provided
21:52:58.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:58.301926 10357 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:52:58.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:58.311992 10357 fs_manager.cc:714] Time spent opening directory manager: real 0.006s	user 0.002s	sys 0.005s
21:52:58.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:58.317241 10374 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:52:58.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:58.318605 10357 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.002s	sys 0.003s
21:52:58.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:58.318950 10357 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-1/data,/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-1/wal
21:52:58.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "29c607b525464ef1bf7fc15526b3a056"
21:52:58.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:52:52 on dist-test-slave-q7qm"
21:52:58.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:58.321043 10357 fs_report.cc:389] FS layout report
21:52:58.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:52:58.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-1/wal
21:52:58.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-1/wal
21:52:58.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-1/data/data
21:52:58.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:52:58.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:52:58.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:52:58.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:52:58.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:52:58.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:52:58.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:52:58.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:52:58.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:52:58.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:52:58.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:52:58.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:58.387060 10357 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:52:58.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:58.399677 10357 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:52:58.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:58.400127 10357 kserver.cc:163] Server-wide thread pool size limit: 3276
21:52:58.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:58.480345 10357 rpc_server.cc:307] RPC server started. Bound to: 127.9.210.61:38683
21:52:58.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:58.480451 10425 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.210.61:38683 every 8 connection(s)
21:52:58.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:58.483347 10357 server_base.cc:1179] Dumped server information to /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-1/data/info.pb
21:52:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:58.489043 10056 external_mini_cluster.cc:1428] Started /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu as pid 10357
21:52:58.489 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:52:58.489 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.9.210.62:42721
21:52:58.490 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: MASTER
    index: 0
  }
}

21:52:58.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:58.491467 10056 external_mini_cluster.cc:1366] Running /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu
21:52:58.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu
21:52:58.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-0/wal
21:52:58.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-0/data
21:52:58.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:52:58.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:52:58.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:52:58.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:52:58.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:52:58.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:52:58.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-0/logs
21:52:58.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-0/data/info.pb
21:52:58.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:52:58.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:52:58.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:52:58.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:52:58.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:52:58.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:52:58.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:52:58.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:52:58.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:52:58.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:52:58.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:52:58.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:52:58.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.210.62:42721
21:52:58.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.210.62
21:52:58.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46869
21:52:58.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/krb5kdc/kudu.keytab
21:52:58.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.210.62
21:52:58.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:52:58.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:52:58.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:52:58.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:52:58.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.210.20:37705
21:52:58.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:52:58.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:52:58.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:52:58.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:52:58.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.210.62:42721,127.9.210.61:38683,127.9.210.60:36675
21:52:58.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:52:58.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42267/jwks.json
21:52:58.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
21:52:58.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:52:58.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:58.495297 10426 sys_catalog.cc:263] Verifying existing consensus state
21:52:58.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20250902 21:52:58.496505 10426 master.cc:416] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
21:52:58.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:58.497076 10357 master.cc:561] Master@127.9.210.61:38683 shutting down...
21:52:58.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:58.575143 10357 master.cc:583] Master@127.9.210.61:38683 shutdown complete.
21:52:58.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:58.577983 10357 builtin_ntp.cc:688] coult not shutdown socket: Network error: shutdown error: Transport endpoint is not connected (error 107)
21:52:58.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
21:52:58.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:58.857591 10428 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:52:58.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:58.858258 10428 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:52:58.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:58.859323 10428 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:52:58.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:58.896978 10428 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:52:58.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:58.897272 10428 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:52:58.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:58.897692 10428 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:52:58.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:58.897974 10428 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:52:58.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:58.898264 10428 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:52:58.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:58.898818 10428 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:52:58.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:58.899109 10428 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42267/jwks.json
21:52:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:58.943572 10428 master_runner.cc:386] Master server non-default flags:
21:52:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:52:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.210.20:37705
21:52:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:52:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:52:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-0/data
21:52:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-0/wal
21:52:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:52:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:52:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.210.62:42721,127.9.210.61:38683,127.9.210.60:36675
21:52:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/krb5kdc/kudu.keytab
21:52:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.210.62
21:52:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:52:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:52:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:52:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.210.62:42721
21:52:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:52:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:52:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:52:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42267/jwks.json
21:52:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:52:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:52:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:52:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:52:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:52:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-0/data/info.pb
21:52:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.210.62
21:52:58.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46869
21:52:58.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:52:58.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:52:58.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:52:58.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:52:58.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:52:58.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-0/logs
21:52:58.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:52:58.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:52:58.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:52:58.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:52:58.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:52:58.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6ab3f249351a4ee5d73eb8bda3585c4dcd85d070
21:52:58.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:52:58.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 21:30:03 UTC on 5fd53c4cbb9d
21:52:58.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7869
21:52:58.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:52:58.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:58.945477 10428 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:52:58.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:52:58.946699 10428 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:52:58.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:58.960736 10435 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:53:00.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:00.363476 10434 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 10428
21:53:00.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:00.823720 10428 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.861s	user 0.628s	sys 1.123s
21:53:00.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:00.826109 10437 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1863 milliseconds
21:53:00.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:00.826172 10428 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.863s	user 0.629s	sys 1.124s
21:53:00.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:52:58.962652 10436 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:53:00.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:00.827286 10428 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:53:00.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:00.827392 10438 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:53:00.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:00.836851 10428 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:53:00.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:00.839695 10428 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:53:00.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:00.841135 10428 hybrid_clock.cc:648] HybridClock initialized: now 1756849980841082 us; error 65 us; skew 500 ppm
21:53:00.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:00.842252 10428 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:53:00.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:53:00 dist-test-slave-q7qm krb5kdc[10067](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849980, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.210.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:53:00.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:00.849143 10428 init.cc:377] Logged in from keytab as kudu/127.9.210.62@KRBTEST.COM (short username kudu)
21:53:00.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:00.854329 10428 webserver.cc:480] Webserver started at http://127.9.210.62:46869/ using document root <none> and password file <none>
21:53:00.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:00.855279 10428 fs_manager.cc:362] Metadata directory not provided
21:53:00.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:00.855576 10428 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:53:00.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:00.864886 10428 fs_manager.cc:714] Time spent opening directory manager: real 0.006s	user 0.006s	sys 0.002s
21:53:00.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:00.870710 10448 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:53:00.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:00.872246 10428 fs_manager.cc:730] Time spent opening block manager: real 0.005s	user 0.005s	sys 0.001s
21:53:00.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:00.872604 10428 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-0/data,/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-0/wal
21:53:00.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6b28dbbfdb35431095e95733b2e72246"
21:53:00.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:52:50 on dist-test-slave-q7qm"
21:53:00.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:00.874769 10428 fs_report.cc:389] FS layout report
21:53:00.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:53:00.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-0/wal
21:53:00.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-0/wal
21:53:00.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-0/data/data
21:53:00.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:53:00.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:53:00.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:53:00.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:53:00.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:53:00.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:53:00.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:53:00.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:53:00.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:53:00.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:53:00.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:53:00.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:00.957330 10428 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:53:00.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:00.963660 10428 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:53:00.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:00.964079 10428 kserver.cc:163] Server-wide thread pool size limit: 3276
21:53:01.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:01.041534 10428 rpc_server.cc:307] RPC server started. Bound to: 127.9.210.62:42721
21:53:01.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:01.041602 10499 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.210.62:42721 every 8 connection(s)
21:53:01.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:01.045106 10428 server_base.cc:1179] Dumped server information to /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-0/data/info.pb
21:53:01.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:01.053925 10056 external_mini_cluster.cc:1428] Started /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu as pid 10428
21:53:01.054 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:53:01.054 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.9.210.60:36675
21:53:01.054 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: MASTER
    index: 2
  }
}

21:53:01.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:01.056309 10056 external_mini_cluster.cc:1366] Running /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu
21:53:01.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu
21:53:01.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-2/wal
21:53:01.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-2/data
21:53:01.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:53:01.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:53:01.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:53:01.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:53:01.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:53:01.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:53:01.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-2/logs
21:53:01.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-2/data/info.pb
21:53:01.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:53:01.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:53:01.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:53:01.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:53:01.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:53:01.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:53:01.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:53:01.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:53:01.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:53:01.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:53:01.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:53:01.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:53:01.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.210.60:36675
21:53:01.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.210.60
21:53:01.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33869
21:53:01.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/krb5kdc/kudu.keytab
21:53:01.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.210.60
21:53:01.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:53:01.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:53:01.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:53:01.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:53:01.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.210.20:37705
21:53:01.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:53:01.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:53:01.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:53:01.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:53:01.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.210.62:42721,127.9.210.61:38683,127.9.210.60:36675
21:53:01.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:53:01.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42267/jwks.json
21:53:01.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
21:53:01.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:53:01.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:01.060258 10500 sys_catalog.cc:263] Verifying existing consensus state
21:53:01.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20250902 21:53:01.061154 10500 master.cc:416] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
21:53:01.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:01.061609 10428 master.cc:561] Master@127.9.210.62:42721 shutting down...
21:53:01.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:01.133143 10428 master.cc:583] Master@127.9.210.62:42721 shutdown complete.
21:53:01.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:01.136608 10428 builtin_ntp.cc:688] coult not shutdown socket: Network error: shutdown error: Transport endpoint is not connected (error 107)
21:53:01.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
21:53:01.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:01.441357 10502 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:53:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:01.441936 10502 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:53:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:01.442696 10502 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:53:01.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:01.477522 10502 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:53:01.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:01.477769 10502 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:53:01.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:01.478035 10502 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:53:01.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:01.478236 10502 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:53:01.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:01.478437 10502 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:53:01.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:01.478627 10502 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:53:01.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:01.478775 10502 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42267/jwks.json
21:53:01.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:01.522322 10502 master_runner.cc:386] Master server non-default flags:
21:53:01.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:53:01.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.210.20:37705
21:53:01.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:53:01.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:53:01.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-2/data
21:53:01.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-2/wal
21:53:01.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:53:01.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:53:01.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.210.62:42721,127.9.210.61:38683,127.9.210.60:36675
21:53:01.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/krb5kdc/kudu.keytab
21:53:01.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.210.60
21:53:01.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:53:01.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:53:01.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:53:01.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.210.60:36675
21:53:01.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:53:01.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:53:01.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:53:01.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42267/jwks.json
21:53:01.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:53:01.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:53:01.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:53:01.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:53:01.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:53:01.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-2/data/info.pb
21:53:01.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.210.60
21:53:01.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33869
21:53:01.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:53:01.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:53:01.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:53:01.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:53:01.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:53:01.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-2/logs
21:53:01.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:53:01.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:53:01.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:53:01.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:53:01.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:53:01.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6ab3f249351a4ee5d73eb8bda3585c4dcd85d070
21:53:01.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:53:01.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 21:30:03 UTC on 5fd53c4cbb9d
21:53:01.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7869
21:53:01.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:53:01.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:01.524031 10502 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:53:01.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:01.525089 10502 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:53:01.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:01.537331 10509 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:53:03.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:02.940291 10508 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 10502
21:53:03.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:01.538582 10510 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:53:03.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:03.627900 10502 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.091s	user 0.745s	sys 1.328s
21:53:03.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:03.628341 10502 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.091s	user 0.745s	sys 1.328s
21:53:03.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:03.631431 10512 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:53:03.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:03.642835 10511 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2104 milliseconds
21:53:03.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:03.642879 10502 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:53:03.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:03.644511 10502 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:53:03.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:03.647549 10502 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:53:03.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:03.648972 10502 hybrid_clock.cc:648] HybridClock initialized: now 1756849983648919 us; error 64 us; skew 500 ppm
21:53:03.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:03.649904 10502 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:53:03.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:53:03 dist-test-slave-q7qm krb5kdc[10067](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849983, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.210.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:53:03.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:03.656563 10502 init.cc:377] Logged in from keytab as kudu/127.9.210.60@KRBTEST.COM (short username kudu)
21:53:03.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:03.661917 10502 webserver.cc:480] Webserver started at http://127.9.210.60:33869/ using document root <none> and password file <none>
21:53:03.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:03.662899 10502 fs_manager.cc:362] Metadata directory not provided
21:53:03.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:03.663154 10502 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:53:03.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:03.671918 10502 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.007s	sys 0.000s
21:53:03.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:03.677201 10522 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:53:03.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:03.678661 10502 fs_manager.cc:730] Time spent opening block manager: real 0.005s	user 0.003s	sys 0.002s
21:53:03.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:03.679121 10502 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-2/data,/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-2/wal
21:53:03.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6a711b14746d4eb6baba23e55bdaac69"
21:53:03.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:52:54 on dist-test-slave-q7qm"
21:53:03.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:03.681668 10502 fs_report.cc:389] FS layout report
21:53:03.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:53:03.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-2/wal
21:53:03.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-2/wal
21:53:03.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-2/data/data
21:53:03.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:53:03.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:53:03.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:53:03.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:53:03.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:53:03.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:53:03.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:53:03.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:53:03.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:53:03.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:53:03.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:53:03.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:03.740240 10502 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:53:03.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:03.758983 10502 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:53:03.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:03.759470 10502 kserver.cc:163] Server-wide thread pool size limit: 3276
21:53:03.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:03.844448 10502 rpc_server.cc:307] RPC server started. Bound to: 127.9.210.60:36675
21:53:03.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:03.845124 10573 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.210.60:36675 every 8 connection(s)
21:53:03.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:03.848052 10502 server_base.cc:1179] Dumped server information to /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/master-2/data/info.pb
21:53:03.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:03.852063 10056 external_mini_cluster.cc:1428] Started /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu as pid 10502
21:53:03.852 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:53:03.855 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.9.210.61:38683(127.9.210.61:38683)] connecting to peer
21:53:03.857 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.9.210.62:42721(127.9.210.62:42721)] connecting to peer
21:53:03.858 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.9.210.60:36675(127.9.210.60:36675)] connecting to peer
21:53:03.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:03.863525 10574 sys_catalog.cc:263] Verifying existing consensus state
21:53:03.868 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:03.869 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.210.62:42721(127.9.210.62:42721)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:03.871 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.62:42721: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:03.872 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:03.873 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.210.61:38683(127.9.210.61:38683)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:03.873 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.61:38683: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:03.873 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.9.210.60:36675(127.9.210.60:36675)] Successfully connected to peer
21:53:03.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:03.873446 10574 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69: Bootstrap starting.
21:53:03.882 [DEBUG - kudu-nio-0] (SecurityContext.java:222) Refreshing Kerberos credentials...
21:53:03.883 [DEBUG - kudu-nio-0] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster2801761017506316466/krb5kdc/krb5cc
21:53:03.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:53:03 dist-test-slave-q7qm krb5kdc[10067](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849975, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:53:03.888 [DEBUG - kudu-nio-0] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
21:53:03.889 [INFO - kudu-nio-0] (SecurityContext.java:256) Successfully refreshed Kerberos credentials from ticket cache
21:53:03.890 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.9.210.60
21:53:03.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:53:03 dist-test-slave-q7qm krb5kdc[10067](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849975, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.9.210.60@KRBTEST.COM
21:53:03.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:03.922907 10574 log.cc:826] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69: Log is configured to *not* fsync() on all Append() calls
21:53:03.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:03.929176 10574 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:53:03.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:03.930121 10574 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69: Bootstrap complete.
21:53:03.933 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xbf9904c2, L:/127.0.0.1:39718 - R:/127.9.210.60:36675] using SASL/GSSAPI
21:53:03.939 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.60:36675: Catalog manager is not initialized. State: Starting
21:53:03.939 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675; will retry
21:53:03.956 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.61:38683(127.9.210.61:38683)] connecting to peer
21:53:03.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:03.954574 10574 raft_consensus.cc:357] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b28dbbfdb35431095e95733b2e72246" member_type: VOTER last_known_addr { host: "127.9.210.62" port: 42721 } } peers { permanent_uuid: "29c607b525464ef1bf7fc15526b3a056" member_type: VOTER last_known_addr { host: "127.9.210.61" port: 38683 } } peers { permanent_uuid: "6a711b14746d4eb6baba23e55bdaac69" member_type: VOTER last_known_addr { host: "127.9.210.60" port: 36675 } }
21:53:03.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:03.957937 10574 raft_consensus.cc:383] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:53:03.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:03.958281 10574 raft_consensus.cc:738] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a711b14746d4eb6baba23e55bdaac69, State: Initialized, Role: FOLLOWER
21:53:03.959 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.62:42721(127.9.210.62:42721)] connecting to peer
21:53:03.960 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:03.960 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.210.61:38683(127.9.210.61:38683)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:03.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:03.959671 10574 consensus_queue.cc:260] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b28dbbfdb35431095e95733b2e72246" member_type: VOTER last_known_addr { host: "127.9.210.62" port: 42721 } } peers { permanent_uuid: "29c607b525464ef1bf7fc15526b3a056" member_type: VOTER last_known_addr { host: "127.9.210.61" port: 38683 } } peers { permanent_uuid: "6a711b14746d4eb6baba23e55bdaac69" member_type: VOTER last_known_addr { host: "127.9.210.60" port: 36675 } }
21:53:03.960 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.61:38683: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:03.964 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:03.964 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.210.62:42721(127.9.210.62:42721)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:03.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:03.963447 10581 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b28dbbfdb35431095e95733b2e72246" member_type: VOTER last_known_addr { host: "127.9.210.62" port: 42721 } } peers { permanent_uuid: "29c607b525464ef1bf7fc15526b3a056" member_type: VOTER last_known_addr { host: "127.9.210.61" port: 38683 } } peers { permanent_uuid: "6a711b14746d4eb6baba23e55bdaac69" member_type: VOTER last_known_addr { host: "127.9.210.60" port: 36675 } } }
21:53:03.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:03.964483 10581 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [sys.catalog]: This master's current role is: FOLLOWER
21:53:03.965 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.62:42721: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:03.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:03.965504 10574 sys_catalog.cc:564] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [sys.catalog]: configured and running, proceeding with master startup.
21:53:03.966 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.60:36675: Catalog manager is not initialized. State: Starting
21:53:03.966 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675; will retry
21:53:03.975 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.61:38683(127.9.210.61:38683)] connecting to peer
21:53:03.976 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.62:42721(127.9.210.62:42721)] connecting to peer
21:53:03.980 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:03.980 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.210.62:42721(127.9.210.62:42721)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:03.981 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.62:42721: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:03.984 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.60:36675: Catalog manager is not initialized. State: Starting
21:53:03.988 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:03.989 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.210.61:38683(127.9.210.61:38683)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:03.989 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.61:38683: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:03.989 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675; will retry
21:53:03.995 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.61:38683(127.9.210.61:38683)] connecting to peer
21:53:03.996 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.62:42721(127.9.210.62:42721)] connecting to peer
21:53:04.000 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:04.000 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.210.61:38683(127.9.210.61:38683)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:04.000 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.61:38683: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:04.004 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:04.004 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.210.62:42721(127.9.210.62:42721)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:04.004 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.62:42721: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:04.008 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675; will retry
21:53:04.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:04.016839 10528 tablet.cc:2378] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:53:04.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:04.020936 10592 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:53:04.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:04.021313 10592 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:53:04.035 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.61:38683(127.9.210.61:38683)] connecting to peer
21:53:04.036 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.62:42721(127.9.210.62:42721)] connecting to peer
21:53:04.039 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:04.039 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.210.62:42721(127.9.210.62:42721)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:04.040 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.62:42721: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:04.040 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:04.040 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.210.61:38683(127.9.210.61:38683)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:04.041 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.61:38683: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:04.048 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675; will retry
21:53:04.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:04.019436 10502 master_runner.cc:186] Error getting master registration for 127.9.210.62:42721: Network error: Client connection negotiation failed: client connection to 127.9.210.62:42721: connect: Connection refused (error 111), 
21:53:04.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:04.061822 10502 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:53:04.075 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.61:38683(127.9.210.61:38683)] connecting to peer
21:53:04.077 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.62:42721(127.9.210.62:42721)] connecting to peer
21:53:04.080 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:04.080 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.210.61:38683(127.9.210.61:38683)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:04.080 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.61:38683: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:04.081 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:04.081 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.210.62:42721(127.9.210.62:42721)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:04.081 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.62:42721: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:04.082 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675; will retry
21:53:04.155 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.61:38683(127.9.210.61:38683)] connecting to peer
21:53:04.157 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.62:42721(127.9.210.62:42721)] connecting to peer
21:53:04.158 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:04.159 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.210.61:38683(127.9.210.61:38683)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:04.159 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.61:38683: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:04.161 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:04.161 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.210.62:42721(127.9.210.62:42721)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:04.161 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.62:42721: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:04.162 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675; will retry
21:53:04.275 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.61:38683(127.9.210.61:38683)] connecting to peer
21:53:04.275 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.62:42721(127.9.210.62:42721)] connecting to peer
21:53:04.280 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:04.280 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.210.62:42721(127.9.210.62:42721)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:04.281 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.62:42721: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:04.283 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:04.283 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.210.61:38683(127.9.210.61:38683)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:04.284 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.61:38683: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:04.284 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675; will retry
21:53:04.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:04.294682 10581 raft_consensus.cc:491] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:53:04.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:04.295244 10581 raft_consensus.cc:513] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b28dbbfdb35431095e95733b2e72246" member_type: VOTER last_known_addr { host: "127.9.210.62" port: 42721 } } peers { permanent_uuid: "29c607b525464ef1bf7fc15526b3a056" member_type: VOTER last_known_addr { host: "127.9.210.61" port: 38683 } } peers { permanent_uuid: "6a711b14746d4eb6baba23e55bdaac69" member_type: VOTER last_known_addr { host: "127.9.210.60" port: 36675 } }
21:53:04.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:04.297982 10581 leader_election.cc:290] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6b28dbbfdb35431095e95733b2e72246 (127.9.210.62:42721), 29c607b525464ef1bf7fc15526b3a056 (127.9.210.61:38683)
21:53:04.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:04.300097 10523 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.210.62:42721: connect: Connection refused (error 111)
21:53:04.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:04.305776 10526 leader_election.cc:336] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 29c607b525464ef1bf7fc15526b3a056 (127.9.210.61:38683): Network error: Client connection negotiation failed: client connection to 127.9.210.61:38683: connect: Connection refused (error 111)
21:53:04.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:04.306360 10523 leader_election.cc:336] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 6b28dbbfdb35431095e95733b2e72246 (127.9.210.62:42721): Network error: Client connection negotiation failed: client connection to 127.9.210.62:42721: connect: Connection refused (error 111)
21:53:04.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:04.306767 10523 leader_election.cc:304] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a711b14746d4eb6baba23e55bdaac69; no voters: 29c607b525464ef1bf7fc15526b3a056, 6b28dbbfdb35431095e95733b2e72246
21:53:04.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:04.307546 10581 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
21:53:04.375 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.61:38683(127.9.210.61:38683)] connecting to peer
21:53:04.376 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.62:42721(127.9.210.62:42721)] connecting to peer
21:53:04.380 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:04.380 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.210.61:38683(127.9.210.61:38683)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:04.380 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:04.380 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.210.62:42721(127.9.210.62:42721)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:04.381 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.62:42721: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:04.381 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.61:38683: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:04.383 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675; will retry
21:53:04.475 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.61:38683(127.9.210.61:38683)] connecting to peer
21:53:04.476 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.62:42721(127.9.210.62:42721)] connecting to peer
21:53:04.478 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:04.478 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.210.61:38683(127.9.210.61:38683)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:04.478 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.61:38683: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:04.480 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:04.480 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.210.62:42721(127.9.210.62:42721)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:04.480 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.62:42721: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:04.481 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675; will retry
21:53:05.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:05.024451 10592 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:53:05.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:05.024756 10592 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:53:05.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:05.035451  9482 debug-util.cc:398] Leaking SignalData structure 0x7b08000adc40 after lost signal to thread 9417
21:53:05.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:05.035996  9482 debug-util.cc:398] Leaking SignalData structure 0x7b0800096140 after lost signal to thread 9485
21:53:05.334 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.61:38683(127.9.210.61:38683)] connecting to peer
21:53:05.335 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.62:42721(127.9.210.62:42721)] connecting to peer
21:53:05.337 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:05.337 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.210.61:38683(127.9.210.61:38683)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:05.337 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.61:38683: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:05.342 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:05.342 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.210.62:42721(127.9.210.62:42721)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:05.343 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.62:42721: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:05.346 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675; will retry
21:53:06.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:06.031742 10592 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:53:06.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:06.032181 10592 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:53:06.275 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.61:38683(127.9.210.61:38683)] connecting to peer
21:53:06.277 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.62:42721(127.9.210.62:42721)] connecting to peer
21:53:06.277 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:06.277 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.210.61:38683(127.9.210.61:38683)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:06.278 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.61:38683: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:06.291 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:06.295 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.210.62:42721(127.9.210.62:42721)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:06.295 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.62:42721: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:06.296 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675; will retry
21:53:06.415 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.61:38683(127.9.210.61:38683)] connecting to peer
21:53:06.417 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:06.417 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.210.61:38683(127.9.210.61:38683)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:06.417 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.61:38683: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:06.418 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.62:42721(127.9.210.62:42721)] connecting to peer
21:53:06.425 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:06.425 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.210.62:42721(127.9.210.62:42721)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:06.425 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.62:42721: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:06.426 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675; will retry
21:53:06.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:06.519946 10599 raft_consensus.cc:491] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:53:06.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:06.520334 10599 raft_consensus.cc:513] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b28dbbfdb35431095e95733b2e72246" member_type: VOTER last_known_addr { host: "127.9.210.62" port: 42721 } } peers { permanent_uuid: "29c607b525464ef1bf7fc15526b3a056" member_type: VOTER last_known_addr { host: "127.9.210.61" port: 38683 } } peers { permanent_uuid: "6a711b14746d4eb6baba23e55bdaac69" member_type: VOTER last_known_addr { host: "127.9.210.60" port: 36675 } }
21:53:06.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:06.521857 10599 leader_election.cc:290] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6b28dbbfdb35431095e95733b2e72246 (127.9.210.62:42721), 29c607b525464ef1bf7fc15526b3a056 (127.9.210.61:38683)
21:53:06.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:06.529819 10526 leader_election.cc:336] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 29c607b525464ef1bf7fc15526b3a056 (127.9.210.61:38683): Network error: Client connection negotiation failed: client connection to 127.9.210.61:38683: connect: Connection refused (error 111)
21:53:06.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:06.530630 10523 leader_election.cc:336] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 6b28dbbfdb35431095e95733b2e72246 (127.9.210.62:42721): Network error: Client connection negotiation failed: client connection to 127.9.210.62:42721: connect: Connection refused (error 111)
21:53:06.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:06.530987 10523 leader_election.cc:304] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a711b14746d4eb6baba23e55bdaac69; no voters: 29c607b525464ef1bf7fc15526b3a056, 6b28dbbfdb35431095e95733b2e72246
21:53:06.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:06.531898 10599 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
21:53:07.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:07.035334 10592 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:53:07.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:07.035703 10592 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:53:08.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:08.038733 10592 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:53:08.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:08.039026 10592 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:53:08.154 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.61:38683(127.9.210.61:38683)] connecting to peer
21:53:08.155 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.62:42721(127.9.210.62:42721)] connecting to peer
21:53:08.157 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:08.157 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.210.61:38683(127.9.210.61:38683)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:08.158 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.61:38683: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:08.159 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:08.159 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.210.62:42721(127.9.210.62:42721)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:08.160 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.62:42721: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:08.161 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675; will retry
21:53:08.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:08.258203 10604 raft_consensus.cc:491] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:53:08.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:08.258584 10604 raft_consensus.cc:513] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b28dbbfdb35431095e95733b2e72246" member_type: VOTER last_known_addr { host: "127.9.210.62" port: 42721 } } peers { permanent_uuid: "29c607b525464ef1bf7fc15526b3a056" member_type: VOTER last_known_addr { host: "127.9.210.61" port: 38683 } } peers { permanent_uuid: "6a711b14746d4eb6baba23e55bdaac69" member_type: VOTER last_known_addr { host: "127.9.210.60" port: 36675 } }
21:53:08.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:08.260361 10604 leader_election.cc:290] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6b28dbbfdb35431095e95733b2e72246 (127.9.210.62:42721), 29c607b525464ef1bf7fc15526b3a056 (127.9.210.61:38683)
21:53:08.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:08.267231 10523 leader_election.cc:336] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 6b28dbbfdb35431095e95733b2e72246 (127.9.210.62:42721): Network error: Client connection negotiation failed: client connection to 127.9.210.62:42721: connect: Connection refused (error 111)
21:53:08.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:08.268009 10526 leader_election.cc:336] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 29c607b525464ef1bf7fc15526b3a056 (127.9.210.61:38683): Network error: Client connection negotiation failed: client connection to 127.9.210.61:38683: connect: Connection refused (error 111)
21:53:08.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:08.268353 10526 leader_election.cc:304] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a711b14746d4eb6baba23e55bdaac69; no voters: 29c607b525464ef1bf7fc15526b3a056, 6b28dbbfdb35431095e95733b2e72246
21:53:08.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:08.269030 10604 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
21:53:09.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:09.042497 10592 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:53:09.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:09.042807 10592 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:53:09.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:09.065218 10502 master_runner.cc:186] Error getting master registration for 127.9.210.62:42721: Network error: Client connection negotiation failed: client connection to 127.9.210.62:42721: connect: Connection refused (error 111), 
21:53:09.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:09.065634 10502 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:53:10.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:10.046888 10592 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:53:10.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:10.047276 10592 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:53:10.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:10.099016 10612 raft_consensus.cc:491] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:53:10.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:10.099529 10612 raft_consensus.cc:513] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b28dbbfdb35431095e95733b2e72246" member_type: VOTER last_known_addr { host: "127.9.210.62" port: 42721 } } peers { permanent_uuid: "29c607b525464ef1bf7fc15526b3a056" member_type: VOTER last_known_addr { host: "127.9.210.61" port: 38683 } } peers { permanent_uuid: "6a711b14746d4eb6baba23e55bdaac69" member_type: VOTER last_known_addr { host: "127.9.210.60" port: 36675 } }
21:53:10.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:10.101152 10612 leader_election.cc:290] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6b28dbbfdb35431095e95733b2e72246 (127.9.210.62:42721), 29c607b525464ef1bf7fc15526b3a056 (127.9.210.61:38683)
21:53:10.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:10.104056 10526 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.210.61:38683: connect: Connection refused (error 111) [suppressed 5 similar messages]
21:53:10.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:10.107725 10523 leader_election.cc:336] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 6b28dbbfdb35431095e95733b2e72246 (127.9.210.62:42721): Network error: Client connection negotiation failed: client connection to 127.9.210.62:42721: connect: Connection refused (error 111)
21:53:10.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:10.108363 10526 leader_election.cc:336] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 29c607b525464ef1bf7fc15526b3a056 (127.9.210.61:38683): Network error: Client connection negotiation failed: client connection to 127.9.210.61:38683: connect: Connection refused (error 111)
21:53:10.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:10.108697 10526 leader_election.cc:304] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a711b14746d4eb6baba23e55bdaac69; no voters: 29c607b525464ef1bf7fc15526b3a056, 6b28dbbfdb35431095e95733b2e72246
21:53:10.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:10.109345 10612 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
21:53:10.614 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.61:38683(127.9.210.61:38683)] connecting to peer
21:53:10.616 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.62:42721(127.9.210.62:42721)] connecting to peer
21:53:10.619 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:10.619 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.210.61:38683(127.9.210.61:38683)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:10.619 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.61:38683: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:10.621 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:10.622 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.210.62:42721(127.9.210.62:42721)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:10.622 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.62:42721: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:10.623 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675; will retry
21:53:10.654 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.61:38683(127.9.210.61:38683)] connecting to peer
21:53:10.655 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.62:42721(127.9.210.62:42721)] connecting to peer
21:53:10.657 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:10.657 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.210.61:38683(127.9.210.61:38683)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:10.658 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:10.658 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.61:38683: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:10.658 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.210.62:42721(127.9.210.62:42721)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:10.658 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.62:42721: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:10.661 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675; will retry
21:53:11.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:11.051299 10592 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:53:11.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:11.051725 10592 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:53:12.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:12.055799 10592 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:53:12.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:12.056159 10592 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:53:12.335 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.61:38683(127.9.210.61:38683)] connecting to peer
21:53:12.336 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.62:42721(127.9.210.62:42721)] connecting to peer
21:53:12.340 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:12.340 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:12.341 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.210.61:38683(127.9.210.61:38683)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:12.341 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.210.62:42721(127.9.210.62:42721)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:12.341 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.62:42721: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:12.341 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.61:38683: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:12.346 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675; will retry
21:53:13.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:13.059732 10592 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:53:13.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:13.060019 10592 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:53:13.174 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.61:38683(127.9.210.61:38683)] connecting to peer
21:53:13.175 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.62:42721(127.9.210.62:42721)] connecting to peer
21:53:13.178 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:13.179 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.210.61:38683(127.9.210.61:38683)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:13.179 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.61:38683: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:13.180 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:13.180 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.210.62:42721(127.9.210.62:42721)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:13.180 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.62:42721: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:13.183 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675; will retry
21:53:13.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:13.307688  9407 debug-util.cc:398] Leaking SignalData structure 0x7b08000943c0 after lost signal to thread 9344
21:53:13.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:13.308650  9407 debug-util.cc:398] Leaking SignalData structure 0x7b08000a91c0 after lost signal to thread 9410
21:53:13.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:53:13 dist-test-slave-q7qm krb5kdc[10067](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849993, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.210.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:53:13.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:13.661111 10518 init.cc:295] Successfully reacquired a new kerberos TGT
21:53:14.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:14.062875 10592 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:53:14.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:14.063174 10592 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:53:14.115 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.61:38683(127.9.210.61:38683)] connecting to peer
21:53:14.116 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.62:42721(127.9.210.62:42721)] connecting to peer
21:53:14.117 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:14.117 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.210.61:38683(127.9.210.61:38683)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:14.118 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.61:38683: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:14.118 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:14.119 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.210.62:42721(127.9.210.62:42721)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:14.119 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.62:42721: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:14.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:14.118326 10502 master_runner.cc:186] Error getting master registration for 127.9.210.62:42721: Network error: Client connection negotiation failed: client connection to 127.9.210.62:42721: connect: Connection refused (error 111), 
21:53:14.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:14.120544 10502 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:53:14.121 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675; will retry
21:53:14.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:14.875738 10570 debug-util.cc:398] Leaking SignalData structure 0x7b0800087f20 after lost signal to thread 10504
21:53:14.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:14.876453 10570 debug-util.cc:398] Leaking SignalData structure 0x7b0800087cc0 after lost signal to thread 10573
21:53:15.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:15.066287 10592 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:53:15.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:15.066608 10592 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:53:16.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:16.069458 10592 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:53:16.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:16.069734 10592 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:53:16.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:16.479305 10625 raft_consensus.cc:491] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:53:16.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:16.479916 10625 raft_consensus.cc:513] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b28dbbfdb35431095e95733b2e72246" member_type: VOTER last_known_addr { host: "127.9.210.62" port: 42721 } } peers { permanent_uuid: "29c607b525464ef1bf7fc15526b3a056" member_type: VOTER last_known_addr { host: "127.9.210.61" port: 38683 } } peers { permanent_uuid: "6a711b14746d4eb6baba23e55bdaac69" member_type: VOTER last_known_addr { host: "127.9.210.60" port: 36675 } }
21:53:16.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:16.481647 10625 leader_election.cc:290] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6b28dbbfdb35431095e95733b2e72246 (127.9.210.62:42721), 29c607b525464ef1bf7fc15526b3a056 (127.9.210.61:38683)
21:53:16.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:16.484490 10523 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.210.62:42721: connect: Connection refused (error 111) [suppressed 1 similar messages]
21:53:16.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:16.488057 10523 leader_election.cc:336] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 6b28dbbfdb35431095e95733b2e72246 (127.9.210.62:42721): Network error: Client connection negotiation failed: client connection to 127.9.210.62:42721: connect: Connection refused (error 111)
21:53:16.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:16.488876 10526 leader_election.cc:336] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 29c607b525464ef1bf7fc15526b3a056 (127.9.210.61:38683): Network error: Client connection negotiation failed: client connection to 127.9.210.61:38683: connect: Connection refused (error 111)
21:53:16.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:16.489259 10526 leader_election.cc:304] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a711b14746d4eb6baba23e55bdaac69; no voters: 29c607b525464ef1bf7fc15526b3a056, 6b28dbbfdb35431095e95733b2e72246
21:53:16.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:16.489934 10625 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
21:53:16.834 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.61:38683(127.9.210.61:38683)] connecting to peer
21:53:16.836 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.62:42721(127.9.210.62:42721)] connecting to peer
21:53:16.838 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:16.838 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.210.61:38683(127.9.210.61:38683)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:16.838 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.61:38683: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:16.839 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:16.839 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.210.62:42721(127.9.210.62:42721)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:16.839 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.62:42721: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:16.841 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675; will retry
21:53:17.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:17.072618 10592 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:53:17.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:17.072911 10592 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:53:17.535 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.61:38683(127.9.210.61:38683)] connecting to peer
21:53:17.536 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.62:42721(127.9.210.62:42721)] connecting to peer
21:53:17.537 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:17.537 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.210.61:38683(127.9.210.61:38683)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:17.537 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.61:38683: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:17.539 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:17.539 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.210.62:42721(127.9.210.62:42721)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:17.539 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.62:42721: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:17.541 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675; will retry
21:53:18.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:18.076188 10592 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:53:18.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:18.076493 10592 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:53:18.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:18.620451 10570 debug-util.cc:398] Leaking SignalData structure 0x7b08000b0060 after lost signal to thread 10504
21:53:18.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:18.621034 10570 debug-util.cc:398] Leaking SignalData structure 0x7b0800087ee0 after lost signal to thread 10573
21:53:19.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:19.079308 10592 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:53:19.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:19.079672 10592 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:53:19.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:19.130522 10502 master_runner.cc:186] Error getting master registration for 127.9.210.62:42721: Network error: Client connection negotiation failed: client connection to 127.9.210.62:42721: connect: Connection refused (error 111), 
21:53:19.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:19.130923 10502 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:53:20.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:20.082952 10592 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:53:20.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:20.083261 10592 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:53:21.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:21.086174 10592 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:53:21.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:21.086535 10592 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:53:21.355 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.61:38683(127.9.210.61:38683)] connecting to peer
21:53:21.356 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.62:42721(127.9.210.62:42721)] connecting to peer
21:53:21.357 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:21.357 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.210.61:38683(127.9.210.61:38683)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:21.358 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.61:38683: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:21.359 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:21.359 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.210.62:42721(127.9.210.62:42721)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:21.359 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.62:42721: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:21.362 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675; will retry
21:53:21.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:21.468214 10041 debug-util.cc:398] Leaking SignalData structure 0x7b08000accc0 after lost signal to thread 9916
21:53:21.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:21.469834 10041 debug-util.cc:398] Leaking SignalData structure 0x7b08000b30a0 after lost signal to thread 10044
21:53:22.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:22.090123 10592 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:53:22.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:22.090543 10592 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:53:22.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:22.582193 10635 raft_consensus.cc:491] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:53:22.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:22.582799 10635 raft_consensus.cc:513] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b28dbbfdb35431095e95733b2e72246" member_type: VOTER last_known_addr { host: "127.9.210.62" port: 42721 } } peers { permanent_uuid: "29c607b525464ef1bf7fc15526b3a056" member_type: VOTER last_known_addr { host: "127.9.210.61" port: 38683 } } peers { permanent_uuid: "6a711b14746d4eb6baba23e55bdaac69" member_type: VOTER last_known_addr { host: "127.9.210.60" port: 36675 } }
21:53:22.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:22.585306 10635 leader_election.cc:290] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6b28dbbfdb35431095e95733b2e72246 (127.9.210.62:42721), 29c607b525464ef1bf7fc15526b3a056 (127.9.210.61:38683)
21:53:22.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:22.589838 10523 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.210.62:42721: connect: Connection refused (error 111) [suppressed 1 similar messages]
21:53:22.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:22.595659 10523 leader_election.cc:336] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 6b28dbbfdb35431095e95733b2e72246 (127.9.210.62:42721): Network error: Client connection negotiation failed: client connection to 127.9.210.62:42721: connect: Connection refused (error 111)
21:53:22.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:22.597519 10526 leader_election.cc:336] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 29c607b525464ef1bf7fc15526b3a056 (127.9.210.61:38683): Network error: Client connection negotiation failed: client connection to 127.9.210.61:38683: connect: Connection refused (error 111)
21:53:22.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:22.601341 10526 leader_election.cc:304] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a711b14746d4eb6baba23e55bdaac69; no voters: 29c607b525464ef1bf7fc15526b3a056, 6b28dbbfdb35431095e95733b2e72246
21:53:22.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:22.602175 10635 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
21:53:23.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:23.093595 10592 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:53:23.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:23.093875 10592 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:53:23.095 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.61:38683(127.9.210.61:38683)] connecting to peer
21:53:23.096 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.62:42721(127.9.210.62:42721)] connecting to peer
21:53:23.097 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:23.097 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.210.61:38683(127.9.210.61:38683)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:23.098 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.61:38683: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:23.099 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:23.099 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.210.62:42721(127.9.210.62:42721)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:23.100 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.62:42721: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:23.103 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675; will retry
21:53:23.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:53:23 dist-test-slave-q7qm krb5kdc[10067](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756850003, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.210.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:53:23.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:23.663959 10518 init.cc:295] Successfully reacquired a new kerberos TGT
21:53:23.835 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.61:38683(127.9.210.61:38683)] connecting to peer
21:53:23.836 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.62:42721(127.9.210.62:42721)] connecting to peer
21:53:23.837 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:23.837 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.210.61:38683(127.9.210.61:38683)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:23.837 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.61:38683: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:23.838 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:23.838 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.210.62:42721(127.9.210.62:42721)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:23.839 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.62:42721: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:23.841 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675; will retry
21:53:24.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:24.097460 10592 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:53:24.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:24.097759 10592 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:53:24.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:24.134183 10502 master_runner.cc:186] Error getting master registration for 127.9.210.62:42721: Network error: Client connection negotiation failed: client connection to 127.9.210.62:42721: connect: Connection refused (error 111), 
21:53:24.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:24.134780 10502 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:53:25.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:25.100673 10592 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:53:25.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:25.101006 10592 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:53:26.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:26.104518 10592 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:53:26.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:26.104831 10592 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:53:27.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:27.108230 10592 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:53:27.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:27.108512 10592 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:53:27.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:27.730165 10647 raft_consensus.cc:491] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:53:27.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:27.730574 10647 raft_consensus.cc:513] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b28dbbfdb35431095e95733b2e72246" member_type: VOTER last_known_addr { host: "127.9.210.62" port: 42721 } } peers { permanent_uuid: "29c607b525464ef1bf7fc15526b3a056" member_type: VOTER last_known_addr { host: "127.9.210.61" port: 38683 } } peers { permanent_uuid: "6a711b14746d4eb6baba23e55bdaac69" member_type: VOTER last_known_addr { host: "127.9.210.60" port: 36675 } }
21:53:27.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:27.732147 10647 leader_election.cc:290] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6b28dbbfdb35431095e95733b2e72246 (127.9.210.62:42721), 29c607b525464ef1bf7fc15526b3a056 (127.9.210.61:38683)
21:53:27.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:27.734941 10523 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.210.62:42721: connect: Connection refused (error 111) [suppressed 1 similar messages]
21:53:27.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:27.738281 10523 leader_election.cc:336] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 6b28dbbfdb35431095e95733b2e72246 (127.9.210.62:42721): Network error: Client connection negotiation failed: client connection to 127.9.210.62:42721: connect: Connection refused (error 111)
21:53:27.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:27.739114 10526 leader_election.cc:336] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 29c607b525464ef1bf7fc15526b3a056 (127.9.210.61:38683): Network error: Client connection negotiation failed: client connection to 127.9.210.61:38683: connect: Connection refused (error 111)
21:53:27.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:27.739488 10526 leader_election.cc:304] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a711b14746d4eb6baba23e55bdaac69; no voters: 29c607b525464ef1bf7fc15526b3a056, 6b28dbbfdb35431095e95733b2e72246
21:53:27.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:27.740065 10647 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
21:53:27.855 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.61:38683(127.9.210.61:38683)] connecting to peer
21:53:27.856 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.62:42721(127.9.210.62:42721)] connecting to peer
21:53:27.857 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:27.857 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.210.61:38683(127.9.210.61:38683)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:27.857 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.61:38683: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:27.858 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:27.858 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.210.62:42721(127.9.210.62:42721)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:27.858 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.62:42721: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:27.860 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675; will retry
21:53:28.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:28.111773 10592 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:53:28.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:28.112061 10592 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:53:28.995 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.61:38683(127.9.210.61:38683)] connecting to peer
21:53:28.995 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.62:42721(127.9.210.62:42721)] connecting to peer
21:53:28.996 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:28.996 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.210.61:38683(127.9.210.61:38683)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:28.996 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.61:38683: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:28.998 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:28.998 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.210.62:42721(127.9.210.62:42721)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:28.999 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.62:42721: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:29.001 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675; will retry
21:53:29.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:29.114832 10592 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:53:29.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:29.115145 10592 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:53:29.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:29.138180 10502 master_runner.cc:186] Error getting master registration for 127.9.210.62:42721: Network error: Client connection negotiation failed: client connection to 127.9.210.62:42721: connect: Connection refused (error 111), 
21:53:29.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:29.138584 10502 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:53:29.875 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.61:38683(127.9.210.61:38683)] connecting to peer
21:53:29.876 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.62:42721(127.9.210.62:42721)] connecting to peer
21:53:29.877 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:29.877 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.210.61:38683(127.9.210.61:38683)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:29.878 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.61:38683: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:29.878 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:29.878 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.210.62:42721(127.9.210.62:42721)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:29.878 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.62:42721: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:29.881 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675; will retry
21:53:30.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:30.118208 10592 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:53:30.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:30.118506 10592 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:53:31.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:31.121493 10592 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:53:31.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:31.121825 10592 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:53:32.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:32.125558 10592 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:53:32.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:32.125898 10592 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:53:32.655 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.61:38683(127.9.210.61:38683)] connecting to peer
21:53:32.656 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.62:42721(127.9.210.62:42721)] connecting to peer
21:53:32.656 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:32.656 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.210.61:38683(127.9.210.61:38683)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:32.656 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.61:38683: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:32.657 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:32.657 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.210.62:42721(127.9.210.62:42721)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:32.657 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.62:42721: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:32.660 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675; will retry
21:53:32.715 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.61:38683(127.9.210.61:38683)] connecting to peer
21:53:32.716 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.62:42721(127.9.210.62:42721)] connecting to peer
21:53:32.716 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:32.717 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.210.61:38683(127.9.210.61:38683)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:32.717 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.61:38683: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:32.717 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:32.718 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.210.62:42721(127.9.210.62:42721)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:32.718 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.62:42721: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:32.720 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675; will retry
21:53:33.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:33.128804 10592 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 6a711b14746d4eb6baba23e55bdaac69: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:53:33.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:33.129102 10592 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:53:33.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:53:33 dist-test-slave-q7qm krb5kdc[10067](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756850013, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.210.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:53:33.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:33.666901 10518 init.cc:295] Successfully reacquired a new kerberos TGT
21:53:33.835 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.61:38683(127.9.210.61:38683)] connecting to peer
21:53:33.835 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.9.210.62:42721(127.9.210.62:42721)] connecting to peer
21:53:33.836 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:33.836 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.210.61:38683(127.9.210.61:38683)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:33.836 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.61:38683: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683
21:53:33.837 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:33.837 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.210.62:42721(127.9.210.62:42721)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:33.837 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.210.62:42721: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
21:53:33.840 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675; will retry
21:53:33.842 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=30, TimeoutTracker(timeout=30000, elapsed=29987), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.61:38683, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.62:42721, [4ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.60:36675, [17ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.62:42721: Network error: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721, [19ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.61:38683: Network error: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683, [84ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.60:36675: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [85ms] delaying RPC due to: Service unavailable: Master config (127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [101ms] refreshing cache from master, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.61:38683, [104ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.62:42721, [106ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.60:36675, [106ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.61:38683: Network error: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683, [110ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.62:42721: Network error: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721, [111ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.60:36675: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [112ms] delaying RPC due to: Service unavailable: Master config (127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.61:38683, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.62:42721, [123ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.60:36675, [127ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.62:42721: Network error: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721, [129ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.60:36675: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [135ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.61:38683: Network error: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683, [136ms] delaying RPC due to: Service unavailable: Master config (127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.61:38683, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.62:42721, [146ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.61:38683: Network error: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683, [148ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.60:36675, [150ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.62:42721: Network error: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721, [154ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.60:36675: OK, [155ms] delaying RPC due to: Service unavailable: Master config (127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.61:38683, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.62:42721, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.60:36675, [186ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.62:42721: Network error: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721, [186ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.61:38683: Network error: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683, [193ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.60:36675: OK, [194ms] delaying RPC due to: Service unavailable: Master config (127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683, [221ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.61:38683, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.62:42721, [223ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.60:36675, [226ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.61:38683: Network error: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683, [227ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.62:42721: Network error: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721, [228ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.60:36675: OK, [228ms] delaying RPC due to: Service unavailable: Master config (127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721, [300ms] refreshing cache from master, [301ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.61:38683, [302ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.62:42721, [304ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.60:36675, [305ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.61:38683: Network error: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683, [307ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.62:42721: Network error: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721, [308ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.60:36675: OK, [309ms] delaying RPC due to: Service unavailable: Master config (127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721, [420ms] refreshing cache from master, [420ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.61:38683, [421ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.62:42721, [422ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.60:36675, [427ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.62:42721: Network error: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721, [430ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.60:36675: OK, [430ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.61:38683: Network error: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683, [430ms] delaying RPC due to: Service unavailable: Master config (127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683, [521ms] refreshing cache from master, [521ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.61:38683, [522ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.62:42721, [523ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.60:36675, [527ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.62:42721: Network error: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721, [527ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.61:38683: Network error: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683, [528ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.60:36675: OK, [529ms] delaying RPC due to: Service unavailable: Master config (127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683, [620ms] refreshing cache from master, [620ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.61:38683, [622ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.62:42721, [623ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.60:36675, [624ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.61:38683: Network error: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683, [626ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.62:42721: Network error: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721, [627ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.60:36675: OK, [628ms] delaying RPC due to: Service unavailable: Master config (127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721, [1480ms] refreshing cache from master, [1480ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.61:38683, [1481ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.62:42721, [1481ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.60:36675, [1483ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.61:38683: Network error: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683, [1489ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.62:42721: Network error: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721, [1491ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.60:36675: OK, [1492ms] delaying RPC due to: Service unavailable: Master config (127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721, [2420ms] refreshing cache from master, [2421ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.61:38683, [2423ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.62:42721, [2423ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.61:38683: Network error: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683, [2424ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.60:36675, [2436ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.60:36675: OK, [2441ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.62:42721: Network error: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721, [2442ms] delaying RPC due to: Service unavailable: Master config (127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721, [2560ms] refreshing cache from master, [2560ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.61:38683, [2563ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.61:38683: Network error: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683, [2564ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.62:42721, [2565ms] too many traces: truncated at 100 traces, deferred=Deferred@973203116(state=PENDING, result=null, callback=(continuation of Deferred@1180775403 after retry RPC after error@1853649837) -> (continuation of Deferred@1089141609 after retry RPC after error@1969963358) -> (continuation of Deferred@987417203 after retry RPC after error@934684574) -> (continuation of Deferred@546350002 after retry RPC after error@257849167) -> (continuation of Deferred@1673248133 after retry RPC after error@1706556603) -> (continuation of Deferred@1084182156 after retry RPC after error@1007140319) -> (continuation of Deferred@40790095 after retry RPC after error@1491472029) -> (continuation of Deferred@1647820835 after retry RPC after error@1667195135) -> (continuation of Deferred@310526155 after retry RPC after error@2054698720) -> (continuation of Deferred@647157588 after retry RPC after error@1053950945) -> (continuation of Deferred@1045858536 after retry RPC after error@86140660) -> (continuation of Deferred@893008243 after retry RPC after error@467269950) -> (continuation of Deferred@297935717 after retry RPC after error@130151023) -> (continuation of Deferred@1674879557 after retry RPC after error@923838216) -> (continuation of Deferred@935949188 after retry RPC after error@1066642840) -> (continuation of Deferred@2031347365 after retry RPC after error@1267279464) -> (continuation of Deferred@933247438 after retry RPC after error@368950735) -> (continuation of Deferred@1933399835 after retry RPC after error@1896191346) -> (continuation of Deferred@1205714133 after retry RPC after error@1060532772) -> (continuation of Deferred@648707926 after retry RPC after error@2069483764) -> (continuation of Deferred@1363734622 after retry RPC after error@1566772993) -> (continuation of Deferred@675711796 after retry RPC after error@489500856) -> (continuation of Deferred@200407363 after retry RPC after error@1311761467) -> (continuation of Deferred@1387055585 after retry RPC after error@644914172) -> (continuation of Deferred@584009104 after retry RPC after error@326132686) -> (continuation of Deferred@665273216 after retry RPC after error@95065075) -> (continuation of Deferred@1569826067 after retry RPC after error@1299174268) -> (continuation of Deferred@1355362834 after retry RPC after error@19732211) -> (continuation of Deferred@1363448568 after retry RPC after error@1644384159), errback=(continuation of Deferred@1180775403 after retry RPC after error@1853649837) -> (continuation of Deferred@1089141609 after retry RPC after error@1969963358) -> (continuation of Deferred@987417203 after retry RPC after error@934684574) -> (continuation of Deferred@546350002 after retry RPC after error@257849167) -> (continuation of Deferred@1673248133 after retry RPC after error@1706556603) -> (continuation of Deferred@1084182156 after retry RPC after error@1007140319) -> (continuation of Deferred@40790095 after retry RPC after error@1491472029) -> (continuation of Deferred@1647820835 after retry RPC after error@1667195135) -> (continuation of Deferred@310526155 after retry RPC after error@2054698720) -> (continuation of Deferred@647157588 after retry RPC after error@1053950945) -> (continuation of Deferred@1045858536 after retry RPC after error@86140660) -> (continuation of Deferred@893008243 after retry RPC after error@467269950) -> (continuation of Deferred@297935717 after retry RPC after error@130151023) -> (continuation of Deferred@1674879557 after retry RPC after error@923838216) -> (continuation of Deferred@935949188 after retry RPC after error@1066642840) -> (continuation of Deferred@2031347365 after retry RPC after error@1267279464) -> (continuation of Deferred@933247438 after retry RPC after error@368950735) -> (continuation of Deferred@1933399835 after retry RPC after error@1896191346) -> (continuation of Deferred@1205714133 after retry RPC after error@1060532772) -> (continuation of Deferred@648707926 after retry RPC after error@2069483764) -> (continuation of Deferred@1363734622 after retry RPC after error@1566772993) -> (continuation of Deferred@675711796 after retry RPC after error@489500856) -> (continuation of Deferred@200407363 after retry RPC after error@1311761467) -> (continuation of Deferred@1387055585 after retry RPC after error@644914172) -> (continuation of Deferred@584009104 after retry RPC after error@326132686) -> (continuation of Deferred@665273216 after retry RPC after error@95065075) -> (continuation of Deferred@1569826067 after retry RPC after error@1299174268) -> (continuation of Deferred@1355362834 after retry RPC after error@19732211) -> (continuation of Deferred@1363448568 after retry RPC after error@1644384159)))
21:53:33.846 [DEBUG - main] (Connection.java:575) [peer master-127.9.210.61:38683(127.9.210.61:38683)] disconnecting while in state TERMINATED
21:53:33.846 [DEBUG - main] (Connection.java:575) [peer master-127.9.210.60:36675(127.9.210.60:36675)] disconnecting while in state READY
21:53:33.846 [DEBUG - main] (Connection.java:575) [peer master-127.9.210.62:42721(127.9.210.62:42721)] disconnecting while in state TERMINATED
21:53:33.850 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
21:53:33.852 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.9.210.60:36675(127.9.210.60:36675)] handling channelInactive
21:53:33.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:33.854029 10056 external_mini_cluster.cc:1658] Killing /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu with pid 10502
21:53:33.862 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.210.60:36675(127.9.210.60:36675)] cleaning up while in state READY due to: connection closed
21:53:35.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-02T21:53:35Z chronyd exiting
21:53:35.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:35.378605  9407 debug-util.cc:398] Leaking SignalData structure 0x7b0800098e40 after lost signal to thread 9344
21:53:35.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:53:35.379462  9407 debug-util.cc:398] Leaking SignalData structure 0x7b08000a8400 after lost signal to thread 9410
21:53:36.204 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
21:53:36.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:36.206844  9331 external_mini_cluster.cc:1658] Killing /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu with pid 9623
21:53:36.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:36.236868  9331 external_mini_cluster.cc:1658] Killing /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu with pid 9769
21:53:36.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:36.265715  9331 external_mini_cluster.cc:1658] Killing /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu with pid 9915
21:53:36.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:36.296307  9331 external_mini_cluster.cc:1658] Killing /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu with pid 9343
21:53:36.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:36.334532  9331 external_mini_cluster.cc:1658] Killing /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu with pid 9413
21:53:36.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:53:36.372236  9331 external_mini_cluster.cc:1658] Killing /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu with pid 9488
21:53:37.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-02T21:53:37Z chronyd exiting
21:53:38.681 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=30, TimeoutTracker(timeout=30000, elapsed=29987), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.61:38683, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.62:42721, [4ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.60:36675, [17ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.62:42721: Network error: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721, [19ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.61:38683: Network error: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683, [84ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.60:36675: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [85ms] delaying RPC due to: Service unavailable: Master config (127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [101ms] refreshing cache from master, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.61:38683, [104ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.62:42721, [106ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.60:36675, [106ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.61:38683: Network error: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683, [110ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.62:42721: Network error: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721, [111ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.60:36675: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [112ms] delaying RPC due to: Service unavailable: Master config (127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.61:38683, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.62:42721, [123ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.60:36675, [127ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.62:42721: Network error: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721, [129ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.60:36675: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [135ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.61:38683: Network error: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683, [136ms] delaying RPC due to: Service unavailable: Master config (127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.61:38683, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.62:42721, [146ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.61:38683: Network error: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683, [148ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.60:36675, [150ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.62:42721: Network error: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721, [154ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.60:36675: OK, [155ms] delaying RPC due to: Service unavailable: Master config (127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.61:38683, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.62:42721, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.60:36675, [186ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.62:42721: Network error: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721, [186ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.61:38683: Network error: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683, [193ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.60:36675: OK, [194ms] delaying RPC due to: Service unavailable: Master config (127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683, [221ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.61:38683, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.62:42721, [223ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.60:36675, [226ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.61:38683: Network error: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683, [227ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.62:42721: Network error: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721, [228ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.60:36675: OK, [228ms] delaying RPC due to: Service unavailable: Master config (127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721, [300ms] refreshing cache from master, [301ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.61:38683, [302ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.62:42721, [304ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.60:36675, [305ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.61:38683: Network error: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683, [307ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.62:42721: Network error: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721, [308ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.60:36675: OK, [309ms] delaying RPC due to: Service unavailable: Master config (127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721, [420ms] refreshing cache from master, [420ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.61:38683, [421ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.62:42721, [422ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.60:36675, [427ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.62:42721: Network error: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721, [430ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.60:36675: OK, [430ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.61:38683: Network error: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683, [430ms] delaying RPC due to: Service unavailable: Master config (127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683, [521ms] refreshing cache from master, [521ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.61:38683, [522ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.62:42721, [523ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.60:36675, [527ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.62:42721: Network error: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721, [527ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.61:38683: Network error: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683, [528ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.60:36675: OK, [529ms] delaying RPC due to: Service unavailable: Master config (127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683, [620ms] refreshing cache from master, [620ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.61:38683, [622ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.62:42721, [623ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.60:36675, [624ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.61:38683: Network error: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683, [626ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.62:42721: Network error: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721, [627ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.60:36675: OK, [628ms] delaying RPC due to: Service unavailable: Master config (127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721, [1480ms] refreshing cache from master, [1480ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.61:38683, [1481ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.62:42721, [1481ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.60:36675, [1483ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.61:38683: Network error: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683, [1489ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.62:42721: Network error: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721, [1491ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.60:36675: OK, [1492ms] delaying RPC due to: Service unavailable: Master config (127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721, [2420ms] refreshing cache from master, [2421ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.61:38683, [2423ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.62:42721, [2423ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.61:38683: Network error: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683, [2424ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.60:36675, [2436ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.60:36675: OK, [2441ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.62:42721: Network error: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721, [2442ms] delaying RPC due to: Service unavailable: Master config (127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721, [2560ms] refreshing cache from master, [2560ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.61:38683, [2563ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.61:38683: Network error: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683, [2564ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.62:42721, [2565ms] too many traces: truncated at 100 traces, deferred=Deferred@973203116(state=PENDING, result=null, callback=(continuation of Deferred@1180775403 after retry RPC after error@1853649837) -> (continuation of Deferred@1089141609 after retry RPC after error@1969963358) -> (continuation of Deferred@987417203 after retry RPC after error@934684574) -> (continuation of Deferred@546350002 after retry RPC after error@257849167) -> (continuation of Deferred@1673248133 after retry RPC after error@1706556603) -> (continuation of Deferred@1084182156 after retry RPC after error@1007140319) -> (continuation of Deferred@40790095 after retry RPC after error@1491472029) -> (continuation of Deferred@1647820835 after retry RPC after error@1667195135) -> (continuation of Deferred@310526155 after retry RPC after error@2054698720) -> (continuation of Deferred@647157588 after retry RPC after error@1053950945) -> (continuation of Deferred@1045858536 after retry RPC after error@86140660) -> (continuation of Deferred@893008243 after retry RPC after error@467269950) -> (continuation of Deferred@297935717 after retry RPC after error@130151023) -> (continuation of Deferred@1674879557 after retry RPC after error@923838216) -> (continuation of Deferred@935949188 after retry RPC after error@1066642840) -> (continuation of Deferred@2031347365 after retry RPC after error@1267279464) -> (continuation of Deferred@933247438 after retry RPC after error@368950735) -> (continuation of Deferred@1933399835 after retry RPC after error@1896191346) -> (continuation of Deferred@1205714133 after retry RPC after error@1060532772) -> (continuation of Deferred@648707926 after retry RPC after error@2069483764) -> (continuation of Deferred@1363734622 after retry RPC after error@1566772993) -> (continuation of Deferred@675711796 after retry RPC after error@489500856) -> (continuation of Deferred@200407363 after retry RPC after error@1311761467) -> (continuation of Deferred@1387055585 after retry RPC after error@644914172) -> (continuation of Deferred@584009104 after retry RPC after error@326132686) -> (continuation of Deferred@665273216 after retry RPC after error@95065075) -> (continuation of Deferred@1569826067 after retry RPC after error@1299174268) -> (continuation of Deferred@1355362834 after retry RPC after error@19732211) -> (continuation of Deferred@1363448568 after retry RPC after error@1644384159), errback=(continuation of Deferred@1180775403 after retry RPC after error@1853649837) -> (continuation of Deferred@1089141609 after retry RPC after error@1969963358) -> (continuation of Deferred@987417203 after retry RPC after error@934684574) -> (continuation of Deferred@546350002 after retry RPC after error@257849167) -> (continuation of Deferred@1673248133 after retry RPC after error@1706556603) -> (continuation of Deferred@1084182156 after retry RPC after error@1007140319) -> (continuation of Deferred@40790095 after retry RPC after error@1491472029) -> (continuation of Deferred@1647820835 after retry RPC after error@1667195135) -> (continuation of Deferred@310526155 after retry RPC after error@2054698720) -> (continuation of Deferred@647157588 after retry RPC after error@1053950945) -> (continuation of Deferred@1045858536 after retry RPC after error@86140660) -> (continuation of Deferred@893008243 after retry RPC after error@467269950) -> (continuation of Deferred@297935717 after retry RPC after error@130151023) -> (continuation of Deferred@1674879557 after retry RPC after error@923838216) -> (continuation of Deferred@935949188 after retry RPC after error@1066642840) -> (continuation of Deferred@2031347365 after retry RPC after error@1267279464) -> (continuation of Deferred@933247438 after retry RPC after error@368950735) -> (continuation of Deferred@1933399835 after retry RPC after error@1896191346) -> (continuation of Deferred@1205714133 after retry RPC after error@1060532772) -> (continuation of Deferred@648707926 after retry RPC after error@2069483764) -> (continuation of Deferred@1363734622 after retry RPC after error@1566772993) -> (continuation of Deferred@675711796 after retry RPC after error@489500856) -> (continuation of Deferred@200407363 after retry RPC after error@1311761467) -> (continuation of Deferred@1387055585 after retry RPC after error@644914172) -> (continuation of Deferred@584009104 after retry RPC after error@326132686) -> (continuation of Deferred@665273216 after retry RPC after error@95065075) -> (continuation of Deferred@1569826067 after retry RPC after error@1299174268) -> (continuation of Deferred@1355362834 after retry RPC after error@19732211) -> (continuation of Deferred@1363448568 after retry RPC after error@1644384159)))
	at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
	at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
	at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
	at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
	at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:346) ~[test/:?]
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
	at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
	at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
	at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
	Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
		at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
		at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
		at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
		at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
		at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
		at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
		at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
		at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
		at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
		at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
		at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
		at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
		at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
		at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
		at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
		at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
		at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
		at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
		at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
		at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
		at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1503) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1366) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1415) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
		at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
		at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
		at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
	at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
	at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
	at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
	at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
	at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
	at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
	at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
	at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
	at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
	at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
	at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
	at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1503) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1366) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1415) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
	... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
	at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
	at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
	at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:326) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:342) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
	... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.9.210.62:42721
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_275]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_275]
	at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
	... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=30, TimeoutTracker(timeout=30000, elapsed=29987), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.61:38683, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.62:42721, [4ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.60:36675, [17ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.62:42721: Network error: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721, [19ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.61:38683: Network error: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683, [84ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.60:36675: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [85ms] delaying RPC due to: Service unavailable: Master config (127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [101ms] refreshing cache from master, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.61:38683, [104ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.62:42721, [106ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.60:36675, [106ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.61:38683: Network error: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683, [110ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.62:42721: Network error: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721, [111ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.60:36675: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [112ms] delaying RPC due to: Service unavailable: Master config (127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.61:38683, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.62:42721, [123ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.60:36675, [127ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.62:42721: Network error: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721, [129ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.60:36675: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [135ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.61:38683: Network error: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683, [136ms] delaying RPC due to: Service unavailable: Master config (127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.61:38683, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.62:42721, [146ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.61:38683: Network error: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683, [148ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.60:36675, [150ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.62:42721: Network error: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721, [154ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.60:36675: OK, [155ms] delaying RPC due to: Service unavailable: Master config (127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.61:38683, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.62:42721, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.60:36675, [186ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.62:42721: Network error: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721, [186ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.61:38683: Network error: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683, [193ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.60:36675: OK, [194ms] delaying RPC due to: Service unavailable: Master config (127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683, [221ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.61:38683, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.62:42721, [223ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.60:36675, [226ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.61:38683: Network error: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683, [227ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.62:42721: Network error: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721, [228ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.60:36675: OK, [228ms] delaying RPC due to: Service unavailable: Master config (127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721, [300ms] refreshing cache from master, [301ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.61:38683, [302ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.62:42721, [304ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.60:36675, [305ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.61:38683: Network error: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683, [307ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.62:42721: Network error: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721, [308ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.60:36675: OK, [309ms] delaying RPC due to: Service unavailable: Master config (127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721, [420ms] refreshing cache from master, [420ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.61:38683, [421ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.62:42721, [422ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.60:36675, [427ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.62:42721: Network error: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721, [430ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.60:36675: OK, [430ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.61:38683: Network error: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683, [430ms] delaying RPC due to: Service unavailable: Master config (127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683, [521ms] refreshing cache from master, [521ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.61:38683, [522ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.62:42721, [523ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.60:36675, [527ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.62:42721: Network error: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721, [527ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.61:38683: Network error: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683, [528ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.60:36675: OK, [529ms] delaying RPC due to: Service unavailable: Master config (127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683, [620ms] refreshing cache from master, [620ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.61:38683, [622ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.62:42721, [623ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.60:36675, [624ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.61:38683: Network error: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683, [626ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.62:42721: Network error: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721, [627ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.60:36675: OK, [628ms] delaying RPC due to: Service unavailable: Master config (127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721, [1480ms] refreshing cache from master, [1480ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.61:38683, [1481ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.62:42721, [1481ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.60:36675, [1483ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.61:38683: Network error: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683, [1489ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.62:42721: Network error: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721, [1491ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.60:36675: OK, [1492ms] delaying RPC due to: Service unavailable: Master config (127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721, [2420ms] refreshing cache from master, [2421ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.61:38683, [2423ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.62:42721, [2423ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.61:38683: Network error: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683, [2424ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.60:36675, [2436ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.60:36675: OK, [2441ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.62:42721: Network error: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721, [2442ms] delaying RPC due to: Service unavailable: Master config (127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721, [2560ms] refreshing cache from master, [2560ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.61:38683, [2563ms] Sub RPC ConnectToMaster: received response from server master-127.9.210.61:38683: Network error: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683, [2564ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.210.62:42721, [2565ms] too many traces: truncated at 100 traces, deferred=Deferred@973203116(state=PENDING, result=null, callback=(continuation of Deferred@1180775403 after retry RPC after error@1853649837) -> (continuation of Deferred@1089141609 after retry RPC after error@1969963358) -> (continuation of Deferred@987417203 after retry RPC after error@934684574) -> (continuation of Deferred@546350002 after retry RPC after error@257849167) -> (continuation of Deferred@1673248133 after retry RPC after error@1706556603) -> (continuation of Deferred@1084182156 after retry RPC after error@1007140319) -> (continuation of Deferred@40790095 after retry RPC after error@1491472029) -> (continuation of Deferred@1647820835 after retry RPC after error@1667195135) -> (continuation of Deferred@310526155 after retry RPC after error@2054698720) -> (continuation of Deferred@647157588 after retry RPC after error@1053950945) -> (continuation of Deferred@1045858536 after retry RPC after error@86140660) -> (continuation of Deferred@893008243 after retry RPC after error@467269950) -> (continuation of Deferred@297935717 after retry RPC after error@130151023) -> (continuation of Deferred@1674879557 after retry RPC after error@923838216) -> (continuation of Deferred@935949188 after retry RPC after error@1066642840) -> (continuation of Deferred@2031347365 after retry RPC after error@1267279464) -> (continuation of Deferred@933247438 after retry RPC after error@368950735) -> (continuation of Deferred@1933399835 after retry RPC after error@1896191346) -> (continuation of Deferred@1205714133 after retry RPC after error@1060532772) -> (continuation of Deferred@648707926 after retry RPC after error@2069483764) -> (continuation of Deferred@1363734622 after retry RPC after error@1566772993) -> (continuation of Deferred@675711796 after retry RPC after error@489500856) -> (continuation of Deferred@200407363 after retry RPC after error@1311761467) -> (continuation of Deferred@1387055585 after retry RPC after error@644914172) -> (continuation of Deferred@584009104 after retry RPC after error@326132686) -> (continuation of Deferred@665273216 after retry RPC after error@95065075) -> (continuation of Deferred@1569826067 after retry RPC after error@1299174268) -> (continuation of Deferred@1355362834 after retry RPC after error@19732211) -> (continuation of Deferred@1363448568 after retry RPC after error@1644384159), errback=(continuation of Deferred@1180775403 after retry RPC after error@1853649837) -> (continuation of Deferred@1089141609 after retry RPC after error@1969963358) -> (continuation of Deferred@987417203 after retry RPC after error@934684574) -> (continuation of Deferred@546350002 after retry RPC after error@257849167) -> (continuation of Deferred@1673248133 after retry RPC after error@1706556603) -> (continuation of Deferred@1084182156 after retry RPC after error@1007140319) -> (continuation of Deferred@40790095 after retry RPC after error@1491472029) -> (continuation of Deferred@1647820835 after retry RPC after error@1667195135) -> (continuation of Deferred@310526155 after retry RPC after error@2054698720) -> (continuation of Deferred@647157588 after retry RPC after error@1053950945) -> (continuation of Deferred@1045858536 after retry RPC after error@86140660) -> (continuation of Deferred@893008243 after retry RPC after error@467269950) -> (continuation of Deferred@297935717 after retry RPC after error@130151023) -> (continuation of Deferred@1674879557 after retry RPC after error@923838216) -> (continuation of Deferred@935949188 after retry RPC after error@1066642840) -> (continuation of Deferred@2031347365 after retry RPC after error@1267279464) -> (continuation of Deferred@933247438 after retry RPC after error@368950735) -> (continuation of Deferred@1933399835 after retry RPC after error@1896191346) -> (continuation of Deferred@1205714133 after retry RPC after error@1060532772) -> (continuation of Deferred@648707926 after retry RPC after error@2069483764) -> (continuation of Deferred@1363734622 after retry RPC after error@1566772993) -> (continuation of Deferred@675711796 after retry RPC after error@489500856) -> (continuation of Deferred@200407363 after retry RPC after error@1311761467) -> (continuation of Deferred@1387055585 after retry RPC after error@644914172) -> (continuation of Deferred@584009104 after retry RPC after error@326132686) -> (continuation of Deferred@665273216 after retry RPC after error@95065075) -> (continuation of Deferred@1569826067 after retry RPC after error@1299174268) -> (continuation of Deferred@1355362834 after retry RPC after error@19732211) -> (continuation of Deferred@1363448568 after retry RPC after error@1644384159)))
	at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
	at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
	at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
	at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
	at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:346)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
	at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
	at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
	at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.lang.Thread.run(Thread.java:748)
	Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
		at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
		at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
		at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
		at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
		at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
		at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
		at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
		at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
		at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
		at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
		at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
		at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
		at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
		at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
		at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
		at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
		at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
		at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
		at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
		at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
		at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
		at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
		at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
		at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
		at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
		at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1503)
		at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1366)
		at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1415)
		at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530)
		at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469)
		at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
		at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
		at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
		at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
		at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
		at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
		at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
		at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
		at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
		at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
		at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
		at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
		... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.9.210.61:38683,127.9.210.62:42721,127.9.210.60:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.61:38683(127.9.210.61:38683): Connection refused: /127.9.210.61:38683,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
	at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
	at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
	at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
	at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
	at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
	at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
	at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
	at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
	at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
	at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
	at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
	at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
	at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
	at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
	at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
	at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
	at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
	at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1503)
	at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1366)
	at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1415)
	at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530)
	at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469)
	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
	at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
	at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
	at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
	at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.210.62:42721(127.9.210.62:42721): Connection refused: /127.9.210.62:42721
	at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
	at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
	at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
	at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
	at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
	at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
	at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
	at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
	at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
	at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:326)
	at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:342)
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
	... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.9.210.62:42721
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
	at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336)
	at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339)
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
	at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)