03:47:48.760 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
03:47:48.760 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskqbWxBn/test-tmp
03:47:48.760 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/asan/bin
03:47:48.760 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskqbWxBn/java/build/dist-test/../../../build/asan/bin/kudu, test, mini_cluster, --serialization=pb]
03:47:48.761 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001"
extra_master_flags: "--txn_manager_enabled"
extra_master_flags: "--raft_heartbeat_interval_ms=100"
extra_tserver_flags: "--txn_keepalive_interval_ms=1000"
extra_tserver_flags: "--txn_staleness_tracker_interval_ms=250"
extra_tserver_flags: "--enable_txn_system_client_init=true"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
03:47:49.119 [DEBUG - main] (MiniKuduCluster.java:195) Response:
03:47:49.119 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
03:47:49.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:49.121897 8053 mini_oidc.cc:150] Starting JWKS server
03:47:49.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:49.124944 8053 webserver.cc:469] Webserver started at http://127.0.0.1:34677/ using document root <none> and password file <none>
03:47:49.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:49.125243 8053 mini_oidc.cc:186] Starting OIDC Discovery server
03:47:49.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:49.125913 8053 webserver.cc:469] Webserver started at http://127.0.0.1:33679/ using document root <none> and password file <none>
03:47:49.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-06-25T03:47:49Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
03:47:49.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-06-25T03:47:49Z Disabled control of system clock
03:47:49.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:49.162576 8053 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskqbWxBn/build/asan/bin/kudu
03:47:49.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqbWxBn/build/asan/bin/kudu
03:47:49.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/master-0/wal
03:47:49.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/master-0/data
03:47:49.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:47:49.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:47:49.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:47:49.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:47:49.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:47:49.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:47:49.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/master-0/logs
03:47:49.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/master-0/data/info.pb
03:47:49.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:47:49.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:47:49.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:47:49.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:47:49.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:47:49.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:47:49.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:47:49.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:47:49.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:47:49.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:47:49.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:47:49.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:47:49.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.221.126:43693
03:47:49.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.221.126
03:47:49.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:47:49.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.221.84:35797
03:47:49.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:47:49.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:47:49.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:47:49.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:47:49.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655
03:47:49.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:47:49.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34677/jwks.json
03:47:49.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
03:47:49.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
03:47:49.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:49.516372 8065 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:47:49.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:49.516783 8065 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:47:49.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:49.516867 8065 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:47:49.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:49.525110 8065 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:47:49.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:49.525234 8065 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
03:47:49.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:49.525286 8065 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:47:49.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:49.525365 8065 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:47:49.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:49.525451 8065 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:47:49.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:49.525504 8065 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:47:49.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:49.525554 8065 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34677/jwks.json
03:47:49.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:49.535773 8065 master_runner.cc:386] Master server non-default flags:
03:47:49.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:47:49.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.221.84:35797
03:47:49.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:47:49.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:47:49.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
03:47:49.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/master-0/data
03:47:49.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/master-0/wal
03:47:49.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:47:49.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655
03:47:49.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
03:47:49.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:47:49.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:47:49.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:47:49.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.221.126:43693
03:47:49.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:47:49.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:47:49.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:47:49.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34677/jwks.json
03:47:49.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:47:49.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:47:49.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/master-0/data/info.pb
03:47:49.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.221.126
03:47:49.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:47:49.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:47:49.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:47:49.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:47:49.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:47:49.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:47:49.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/master-0/logs
03:47:49.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:47:49.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:47:49.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:47:49.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:47:49.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:47:49.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6d72d4a242076468501f3430b9a2cd050c634be2
03:47:49.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:47:49.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Jun 2025 03:18:58 UTC on 5fd53c4cbb9d
03:47:49.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6756
03:47:49.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
03:47:49.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:49.537633 8065 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:47:49.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:49.539728 8065 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:47:49.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:49.551440 8073 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:47:49.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:49.551607 8071 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:47:49.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:49.551505 8070 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:47:49.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:49.552640 8065 server_base.cc:1048] running on GCE node
03:47:49.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:49.553610 8065 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:47:49.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:49.555296 8065 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:47:49.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:49.556617 8065 hybrid_clock.cc:648] HybridClock initialized: now 1750823269556499 us; error 43 us; skew 500 ppm
03:47:49.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:49.557040 8065 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:47:49.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:49.559850 8065 webserver.cc:469] Webserver started at http://127.7.221.126:38867/ using document root <none> and password file <none>
03:47:49.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:49.560818 8065 fs_manager.cc:362] Metadata directory not provided
03:47:49.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:49.560972 8065 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:47:49.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:49.561347 8065 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
03:47:49.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:49.564127 8065 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/master-0/data/instance:
03:47:49.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4c24d4441f6c47728564eedff9749c8f"
03:47:49.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 03:47:49 on dist-test-slave-zzn2"
03:47:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:49.565088 8065 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/master-0/wal/instance:
03:47:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4c24d4441f6c47728564eedff9749c8f"
03:47:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 03:47:49 on dist-test-slave-zzn2"
03:47:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:49.571420 8065 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.005s sys 0.000s
03:47:49.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:49.575523 8079 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:47:49.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:49.577395 8065 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
03:47:49.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:49.577639 8065 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/master-0/data,/tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/master-0/wal
03:47:49.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4c24d4441f6c47728564eedff9749c8f"
03:47:49.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 03:47:49 on dist-test-slave-zzn2"
03:47:49.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:49.577877 8065 fs_report.cc:389] FS layout report
03:47:49.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:47:49.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/master-0/wal
03:47:49.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/master-0/wal
03:47:49.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/master-0/data/data
03:47:49.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:47:49.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:47:49.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:47:49.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:47:49.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:47:49.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:47:49.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:47:49.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:47:49.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:47:49.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:47:49.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:47:49.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:49.599521 8065 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:47:49.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:49.600636 8065 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:47:49.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:49.600983 8065 kserver.cc:163] Server-wide thread pool size limit: 3276
03:47:49.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:49.627478 8065 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.126:43693
03:47:49.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:49.627480 8140 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.7.221.126:43693 every 8 connection(s)
03:47:49.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:49.629508 8065 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/master-0/data/info.pb
03:47:49.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:49.633827 8053 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskqbWxBn/build/asan/bin/kudu as pid 8065
03:47:49.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:49.634121 8053 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/master-0/wal/instance
03:47:49.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:49.634480 8141 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:47:49.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:49.636703 8053 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskqbWxBn/build/asan/bin/kudu
03:47:49.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqbWxBn/build/asan/bin/kudu
03:47:49.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/master-1/wal
03:47:49.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/master-1/data
03:47:49.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:47:49.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:47:49.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:47:49.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:47:49.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:47:49.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:47:49.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/master-1/logs
03:47:49.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/master-1/data/info.pb
03:47:49.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:47:49.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:47:49.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:47:49.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:47:49.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:47:49.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:47:49.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:47:49.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:47:49.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:47:49.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:47:49.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:47:49.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:47:49.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.221.125:33545
03:47:49.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.221.125
03:47:49.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:47:49.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.221.84:35797
03:47:49.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:47:49.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:47:49.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:47:49.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:47:49.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655
03:47:49.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:47:49.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34677/jwks.json
03:47:49.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
03:47:49.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
03:47:49.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:49.642046 8141 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.221.126" port: 43693 } has no permanent_uuid. Determining permanent_uuid...
03:47:49.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:49.660609 8141 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.221.125" port: 33545 } has no permanent_uuid. Determining permanent_uuid...
03:47:49.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:49.661921 8083 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.125:33545: connect: Connection refused (error 111)
03:47:49.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:49.663729 8141 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.221.125:33545: Network error: Client connection negotiation failed: client connection to 127.7.221.125:33545: connect: Connection refused (error 111)
03:47:49.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:49.713135 8141 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.221.125" port: 33545 } attempt: 1
03:47:49.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:49.715271 8141 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.221.125:33545: Network error: Client connection negotiation failed: client connection to 127.7.221.125:33545: connect: Connection refused (error 111)
03:47:49.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:49.783598 8141 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.221.125" port: 33545 } attempt: 2
03:47:49.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:49.785629 8141 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.221.125:33545: Network error: Client connection negotiation failed: client connection to 127.7.221.125:33545: connect: Connection refused (error 111)
03:47:49.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:49.877029 8141 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.221.125" port: 33545 } attempt: 3
03:47:49.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:49.878980 8141 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.221.125:33545: Network error: Client connection negotiation failed: client connection to 127.7.221.125:33545: connect: Connection refused (error 111)
03:47:49.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:49.988119 8143 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:47:49.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:49.988579 8143 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:47:49.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:49.988726 8143 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:47:49.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:49.996804 8143 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:47:49.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:49.996939 8143 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
03:47:49.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:49.997025 8143 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:47:49.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:49.997087 8143 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:47:49.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:49.997169 8143 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:47:49.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:49.997229 8143 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:47:49.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:49.997296 8143 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34677/jwks.json
03:47:50.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.007364 8143 master_runner.cc:386] Master server non-default flags:
03:47:50.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:47:50.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.221.84:35797
03:47:50.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:47:50.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:47:50.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
03:47:50.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/master-1/data
03:47:50.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/master-1/wal
03:47:50.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:47:50.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655
03:47:50.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
03:47:50.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:47:50.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:47:50.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:47:50.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.221.125:33545
03:47:50.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:47:50.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:47:50.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:47:50.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34677/jwks.json
03:47:50.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:47:50.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:47:50.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/master-1/data/info.pb
03:47:50.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.221.125
03:47:50.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:47:50.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:47:50.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:47:50.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:47:50.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:47:50.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:47:50.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/master-1/logs
03:47:50.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:47:50.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:47:50.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:47:50.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:47:50.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:47:50.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6d72d4a242076468501f3430b9a2cd050c634be2
03:47:50.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:47:50.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Jun 2025 03:18:58 UTC on 5fd53c4cbb9d
03:47:50.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6756
03:47:50.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
03:47:50.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.009513 8143 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:47:50.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.011567 8143 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:47:50.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.022243 8141 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.221.125" port: 33545 } attempt: 4
03:47:50.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:50.022727 8152 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:47:50.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:50.022912 8154 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:47:50.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:50.023065 8151 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:47:50.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.023912 8143 server_base.cc:1048] running on GCE node
03:47:50.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:50.024549 8141 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.221.125:33545: Network error: Client connection negotiation failed: client connection to 127.7.221.125:33545: connect: Connection refused (error 111)
03:47:50.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.024974 8143 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:47:50.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.026603 8143 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:47:50.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.027875 8143 hybrid_clock.cc:648] HybridClock initialized: now 1750823270027804 us; error 48 us; skew 500 ppm
03:47:50.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.028278 8143 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:47:50.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.030972 8143 webserver.cc:469] Webserver started at http://127.7.221.125:43235/ using document root <none> and password file <none>
03:47:50.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.031914 8143 fs_manager.cc:362] Metadata directory not provided
03:47:50.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.032066 8143 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:47:50.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.032470 8143 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
03:47:50.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.035285 8143 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/master-1/data/instance:
03:47:50.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "abe570d88d9e41929e679edcc29956c5"
03:47:50.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 03:47:50 on dist-test-slave-zzn2"
03:47:50.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.036206 8143 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/master-1/wal/instance:
03:47:50.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "abe570d88d9e41929e679edcc29956c5"
03:47:50.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 03:47:50 on dist-test-slave-zzn2"
03:47:50.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.042654 8143 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.004s sys 0.000s
03:47:50.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.046754 8160 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:47:50.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.048454 8143 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.001s sys 0.000s
03:47:50.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.048681 8143 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/master-1/data,/tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/master-1/wal
03:47:50.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "abe570d88d9e41929e679edcc29956c5"
03:47:50.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 03:47:50 on dist-test-slave-zzn2"
03:47:50.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.048887 8143 fs_report.cc:389] FS layout report
03:47:50.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:47:50.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/master-1/wal
03:47:50.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/master-1/wal
03:47:50.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/master-1/data/data
03:47:50.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:47:50.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:47:50.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:47:50.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:47:50.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:47:50.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:47:50.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:47:50.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:47:50.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:47:50.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:47:50.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:47:50.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.075170 8143 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:47:50.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.076133 8143 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:47:50.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.076406 8143 kserver.cc:163] Server-wide thread pool size limit: 3276
03:47:50.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.102072 8143 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.125:33545
03:47:50.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.102090 8222 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.7.221.125:33545 every 8 connection(s)
03:47:50.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.103840 8143 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/master-1/data/info.pb
03:47:50.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.108513 8223 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:47:50.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.108491 8053 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskqbWxBn/build/asan/bin/kudu as pid 8143
03:47:50.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.108733 8053 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/master-1/wal/instance
03:47:50.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.110049 8053 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskqbWxBn/build/asan/bin/kudu
03:47:50.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqbWxBn/build/asan/bin/kudu
03:47:50.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/master-2/wal
03:47:50.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/master-2/data
03:47:50.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:47:50.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:47:50.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:47:50.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:47:50.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:47:50.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:47:50.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/master-2/logs
03:47:50.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/master-2/data/info.pb
03:47:50.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:47:50.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:47:50.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:47:50.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:47:50.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:47:50.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:47:50.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:47:50.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:47:50.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:47:50.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:47:50.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:47:50.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:47:50.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.221.124:35655
03:47:50.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.221.124
03:47:50.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:47:50.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.221.84:35797
03:47:50.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:47:50.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:47:50.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:47:50.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:47:50.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655
03:47:50.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:47:50.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34677/jwks.json
03:47:50.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
03:47:50.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
03:47:50.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.117484 8223 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.221.126" port: 43693 } has no permanent_uuid. Determining permanent_uuid...
03:47:50.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.133216 8223 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.221.125" port: 33545 } has no permanent_uuid. Determining permanent_uuid...
03:47:50.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.142828 8223 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.221.124" port: 35655 } has no permanent_uuid. Determining permanent_uuid...
03:47:50.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:50.144232 8161 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.124:35655: connect: Connection refused (error 111)
03:47:50.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:50.145948 8223 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.221.124:35655: Network error: Client connection negotiation failed: client connection to 127.7.221.124:35655: connect: Connection refused (error 111)
03:47:50.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.195389 8223 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.221.124" port: 35655 } attempt: 1
03:47:50.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:50.197582 8223 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.221.124:35655: Network error: Client connection negotiation failed: client connection to 127.7.221.124:35655: connect: Connection refused (error 111)
03:47:50.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.265856 8223 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.221.124" port: 35655 } attempt: 2
03:47:50.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:50.267887 8223 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.221.124:35655: Network error: Client connection negotiation failed: client connection to 127.7.221.124:35655: connect: Connection refused (error 111)
03:47:50.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.323913 8141 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.221.125" port: 33545 } attempt: 5
03:47:50.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.331041 8141 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.221.124" port: 35655 } has no permanent_uuid. Determining permanent_uuid...
03:47:50.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:50.333057 8141 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.221.124:35655: Network error: Client connection negotiation failed: client connection to 127.7.221.124:35655: connect: Connection refused (error 111)
03:47:50.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.359158 8223 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.221.124" port: 35655 } attempt: 3
03:47:50.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:50.361533 8223 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.221.124:35655: Network error: Client connection negotiation failed: client connection to 127.7.221.124:35655: connect: Connection refused (error 111)
03:47:50.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.384291 8141 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.221.124" port: 35655 } attempt: 1
03:47:50.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:50.386476 8141 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.221.124:35655: Network error: Client connection negotiation failed: client connection to 127.7.221.124:35655: connect: Connection refused (error 111)
03:47:50.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.454752 8141 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.221.124" port: 35655 } attempt: 2
03:47:50.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:50.457006 8141 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.221.124:35655: Network error: Client connection negotiation failed: client connection to 127.7.221.124:35655: connect: Connection refused (error 111)
03:47:50.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:50.474133 8225 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:47:50.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:50.474538 8225 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:47:50.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:50.474640 8225 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:47:50.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:50.482955 8225 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:47:50.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:50.483121 8225 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
03:47:50.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:50.483210 8225 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:47:50.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:50.483263 8225 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:47:50.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:50.483331 8225 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:47:50.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:50.483395 8225 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:47:50.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:50.483448 8225 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34677/jwks.json
03:47:50.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.493772 8225 master_runner.cc:386] Master server non-default flags:
03:47:50.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:47:50.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.221.84:35797
03:47:50.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:47:50.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:47:50.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
03:47:50.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/master-2/data
03:47:50.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/master-2/wal
03:47:50.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:47:50.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655
03:47:50.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
03:47:50.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:47:50.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:47:50.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:47:50.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.221.124:35655
03:47:50.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:47:50.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:47:50.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:47:50.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34677/jwks.json
03:47:50.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:47:50.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:47:50.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/master-2/data/info.pb
03:47:50.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.221.124
03:47:50.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:47:50.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:47:50.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:47:50.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:47:50.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:47:50.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:47:50.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/master-2/logs
03:47:50.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:47:50.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:47:50.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:47:50.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:47:50.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:47:50.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6d72d4a242076468501f3430b9a2cd050c634be2
03:47:50.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:47:50.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Jun 2025 03:18:58 UTC on 5fd53c4cbb9d
03:47:50.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6756
03:47:50.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
03:47:50.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.495612 8225 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:47:50.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.497881 8225 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:47:50.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.504873 8223 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.221.124" port: 35655 } attempt: 4
03:47:50.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:50.507048 8223 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.221.124:35655: Network error: Client connection negotiation failed: client connection to 127.7.221.124:35655: connect: Connection refused (error 111)
03:47:50.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:50.510175 8234 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:47:50.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:50.510771 8233 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:47:50.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:50.510829 8236 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:47:50.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.510779 8225 server_base.cc:1048] running on GCE node
03:47:50.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.512014 8225 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:47:50.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.513810 8225 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:47:50.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.515095 8225 hybrid_clock.cc:648] HybridClock initialized: now 1750823270515030 us; error 42 us; skew 500 ppm
03:47:50.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.515527 8225 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:47:50.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.523442 8225 webserver.cc:469] Webserver started at http://127.7.221.124:42033/ using document root <none> and password file <none>
03:47:50.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.524385 8225 fs_manager.cc:362] Metadata directory not provided
03:47:50.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.524586 8225 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:47:50.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.524991 8225 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
03:47:50.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.527712 8225 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/master-2/data/instance:
03:47:50.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "59d0ebb2088d4fafbfc147165d7544f8"
03:47:50.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 03:47:50 on dist-test-slave-zzn2"
03:47:50.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.528728 8225 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/master-2/wal/instance:
03:47:50.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "59d0ebb2088d4fafbfc147165d7544f8"
03:47:50.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 03:47:50 on dist-test-slave-zzn2"
03:47:50.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.535040 8225 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.001s sys 0.004s
03:47:50.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.539192 8242 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:47:50.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.541045 8225 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
03:47:50.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.541268 8225 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/master-2/data,/tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/master-2/wal
03:47:50.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "59d0ebb2088d4fafbfc147165d7544f8"
03:47:50.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 03:47:50 on dist-test-slave-zzn2"
03:47:50.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.541507 8225 fs_report.cc:389] FS layout report
03:47:50.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:47:50.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/master-2/wal
03:47:50.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/master-2/wal
03:47:50.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/master-2/data/data
03:47:50.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:47:50.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:47:50.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:47:50.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:47:50.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:47:50.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:47:50.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:47:50.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:47:50.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:47:50.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:47:50.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:47:50.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.557703 8225 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:47:50.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.558707 8225 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:47:50.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.559039 8225 kserver.cc:163] Server-wide thread pool size limit: 3276
03:47:50.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.563244 8141 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.221.124" port: 35655 } attempt: 3
03:47:50.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:50.565191 8141 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.221.124:35655: Network error: Client connection negotiation failed: client connection to 127.7.221.124:35655: connect: Connection refused (error 111)
03:47:50.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.584115 8225 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.124:35655
03:47:50.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.584143 8303 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.7.221.124:35655 every 8 connection(s)
03:47:50.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.585984 8225 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/master-2/data/info.pb
03:47:50.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.590689 8304 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:47:50.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.591976 8053 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskqbWxBn/build/asan/bin/kudu as pid 8225
03:47:50.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.592173 8053 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/master-2/wal/instance
03:47:50.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.597517 8304 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.221.126" port: 43693 } has no permanent_uuid. Determining permanent_uuid...
03:47:50.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.627118 8304 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.221.125" port: 33545 } has no permanent_uuid. Determining permanent_uuid...
03:47:50.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.636021 8304 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.221.124" port: 35655 } has no permanent_uuid. Determining permanent_uuid...
03:47:50.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.647914 8304 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 59d0ebb2088d4fafbfc147165d7544f8: Bootstrap starting.
03:47:50.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.651533 8304 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 59d0ebb2088d4fafbfc147165d7544f8: Neither blocks nor log segments found. Creating new log.
03:47:50.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.653097 8304 log.cc:826] T 00000000000000000000000000000000 P 59d0ebb2088d4fafbfc147165d7544f8: Log is configured to *not* fsync() on all Append() calls
03:47:50.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.656090 8304 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 59d0ebb2088d4fafbfc147165d7544f8: No bootstrap required, opened a new log
03:47:50.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.661958 8304 raft_consensus.cc:357] T 00000000000000000000000000000000 P 59d0ebb2088d4fafbfc147165d7544f8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c24d4441f6c47728564eedff9749c8f" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 43693 } } peers { permanent_uuid: "abe570d88d9e41929e679edcc29956c5" member_type: VOTER last_known_addr { host: "127.7.221.125" port: 33545 } } peers { permanent_uuid: "59d0ebb2088d4fafbfc147165d7544f8" member_type: VOTER last_known_addr { host: "127.7.221.124" port: 35655 } }
03:47:50.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.662395 8304 raft_consensus.cc:383] T 00000000000000000000000000000000 P 59d0ebb2088d4fafbfc147165d7544f8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:47:50.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.662504 8304 raft_consensus.cc:738] T 00000000000000000000000000000000 P 59d0ebb2088d4fafbfc147165d7544f8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 59d0ebb2088d4fafbfc147165d7544f8, State: Initialized, Role: FOLLOWER
03:47:50.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.663303 8304 consensus_queue.cc:260] T 00000000000000000000000000000000 P 59d0ebb2088d4fafbfc147165d7544f8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c24d4441f6c47728564eedff9749c8f" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 43693 } } peers { permanent_uuid: "abe570d88d9e41929e679edcc29956c5" member_type: VOTER last_known_addr { host: "127.7.221.125" port: 33545 } } peers { permanent_uuid: "59d0ebb2088d4fafbfc147165d7544f8" member_type: VOTER last_known_addr { host: "127.7.221.124" port: 35655 } }
03:47:50.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.664420 8311 sys_catalog.cc:455] T 00000000000000000000000000000000 P 59d0ebb2088d4fafbfc147165d7544f8 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c24d4441f6c47728564eedff9749c8f" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 43693 } } peers { permanent_uuid: "abe570d88d9e41929e679edcc29956c5" member_type: VOTER last_known_addr { host: "127.7.221.125" port: 33545 } } peers { permanent_uuid: "59d0ebb2088d4fafbfc147165d7544f8" member_type: VOTER last_known_addr { host: "127.7.221.124" port: 35655 } } }
03:47:50.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.664785 8311 sys_catalog.cc:458] T 00000000000000000000000000000000 P 59d0ebb2088d4fafbfc147165d7544f8 [sys.catalog]: This master's current role is: FOLLOWER
03:47:50.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.665647 8304 sys_catalog.cc:564] T 00000000000000000000000000000000 P 59d0ebb2088d4fafbfc147165d7544f8 [sys.catalog]: configured and running, proceeding with master startup.
03:47:50.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.673810 8311 raft_consensus.cc:491] T 00000000000000000000000000000000 P 59d0ebb2088d4fafbfc147165d7544f8 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:47:50.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.674156 8311 raft_consensus.cc:513] T 00000000000000000000000000000000 P 59d0ebb2088d4fafbfc147165d7544f8 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c24d4441f6c47728564eedff9749c8f" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 43693 } } peers { permanent_uuid: "abe570d88d9e41929e679edcc29956c5" member_type: VOTER last_known_addr { host: "127.7.221.125" port: 33545 } } peers { permanent_uuid: "59d0ebb2088d4fafbfc147165d7544f8" member_type: VOTER last_known_addr { host: "127.7.221.124" port: 35655 } }
03:47:50.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.676244 8311 leader_election.cc:290] T 00000000000000000000000000000000 P 59d0ebb2088d4fafbfc147165d7544f8 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4c24d4441f6c47728564eedff9749c8f (127.7.221.126:43693), abe570d88d9e41929e679edcc29956c5 (127.7.221.125:33545)
03:47:50.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:50.676990 8322 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 59d0ebb2088d4fafbfc147165d7544f8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:47:50.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:50.677134 8322 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:47:50.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.674026 8225 master_runner.cc:186] Error getting master registration for 127.7.221.126:43693: OK, instance_id { permanent_uuid: "4c24d4441f6c47728564eedff9749c8f" instance_seqno: 1750823269620765 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:47:50.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.682749 8225 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:47:50.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.742447 8141 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.221.124" port: 35655 } attempt: 4
03:47:50.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.755861 8141 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4c24d4441f6c47728564eedff9749c8f: Bootstrap starting.
03:47:50.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.760597 8141 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 4c24d4441f6c47728564eedff9749c8f: Neither blocks nor log segments found. Creating new log.
03:47:50.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.762588 8141 log.cc:826] T 00000000000000000000000000000000 P 4c24d4441f6c47728564eedff9749c8f: Log is configured to *not* fsync() on all Append() calls
03:47:50.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.772931 8141 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4c24d4441f6c47728564eedff9749c8f: No bootstrap required, opened a new log
03:47:50.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.782785 8141 raft_consensus.cc:357] T 00000000000000000000000000000000 P 4c24d4441f6c47728564eedff9749c8f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c24d4441f6c47728564eedff9749c8f" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 43693 } } peers { permanent_uuid: "abe570d88d9e41929e679edcc29956c5" member_type: VOTER last_known_addr { host: "127.7.221.125" port: 33545 } } peers { permanent_uuid: "59d0ebb2088d4fafbfc147165d7544f8" member_type: VOTER last_known_addr { host: "127.7.221.124" port: 35655 } }
03:47:50.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.783258 8141 raft_consensus.cc:383] T 00000000000000000000000000000000 P 4c24d4441f6c47728564eedff9749c8f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:47:50.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.783377 8141 raft_consensus.cc:738] T 00000000000000000000000000000000 P 4c24d4441f6c47728564eedff9749c8f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4c24d4441f6c47728564eedff9749c8f, State: Initialized, Role: FOLLOWER
03:47:50.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.784355 8141 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4c24d4441f6c47728564eedff9749c8f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c24d4441f6c47728564eedff9749c8f" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 43693 } } peers { permanent_uuid: "abe570d88d9e41929e679edcc29956c5" member_type: VOTER last_known_addr { host: "127.7.221.125" port: 33545 } } peers { permanent_uuid: "59d0ebb2088d4fafbfc147165d7544f8" member_type: VOTER last_known_addr { host: "127.7.221.124" port: 35655 } }
03:47:50.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.786077 8325 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4c24d4441f6c47728564eedff9749c8f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c24d4441f6c47728564eedff9749c8f" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 43693 } } peers { permanent_uuid: "abe570d88d9e41929e679edcc29956c5" member_type: VOTER last_known_addr { host: "127.7.221.125" port: 33545 } } peers { permanent_uuid: "59d0ebb2088d4fafbfc147165d7544f8" member_type: VOTER last_known_addr { host: "127.7.221.124" port: 35655 } } }
03:47:50.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.786665 8325 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4c24d4441f6c47728564eedff9749c8f [sys.catalog]: This master's current role is: FOLLOWER
03:47:50.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.787501 8141 sys_catalog.cc:564] T 00000000000000000000000000000000 P 4c24d4441f6c47728564eedff9749c8f [sys.catalog]: configured and running, proceeding with master startup.
03:47:50.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.787549 8106 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "59d0ebb2088d4fafbfc147165d7544f8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4c24d4441f6c47728564eedff9749c8f" is_pre_election: true
03:47:50.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.788185 8106 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 4c24d4441f6c47728564eedff9749c8f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 59d0ebb2088d4fafbfc147165d7544f8 in term 0.
03:47:50.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.789110 8246 leader_election.cc:304] T 00000000000000000000000000000000 P 59d0ebb2088d4fafbfc147165d7544f8 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4c24d4441f6c47728564eedff9749c8f, 59d0ebb2088d4fafbfc147165d7544f8; no voters:
03:47:50.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.789736 8311 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 59d0ebb2088d4fafbfc147165d7544f8 [term 0 FOLLOWER]: Leader pre-election won for term 1
03:47:50.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.789932 8311 raft_consensus.cc:491] T 00000000000000000000000000000000 P 59d0ebb2088d4fafbfc147165d7544f8 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:47:50.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.790047 8311 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 59d0ebb2088d4fafbfc147165d7544f8 [term 0 FOLLOWER]: Advancing to term 1
03:47:50.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.793752 8311 raft_consensus.cc:513] T 00000000000000000000000000000000 P 59d0ebb2088d4fafbfc147165d7544f8 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c24d4441f6c47728564eedff9749c8f" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 43693 } } peers { permanent_uuid: "abe570d88d9e41929e679edcc29956c5" member_type: VOTER last_known_addr { host: "127.7.221.125" port: 33545 } } peers { permanent_uuid: "59d0ebb2088d4fafbfc147165d7544f8" member_type: VOTER last_known_addr { host: "127.7.221.124" port: 35655 } }
03:47:50.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.795409 8106 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "59d0ebb2088d4fafbfc147165d7544f8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4c24d4441f6c47728564eedff9749c8f"
03:47:50.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.795709 8106 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 4c24d4441f6c47728564eedff9749c8f [term 0 FOLLOWER]: Advancing to term 1
03:47:50.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.803781 8106 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 4c24d4441f6c47728564eedff9749c8f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 59d0ebb2088d4fafbfc147165d7544f8 in term 1.
03:47:50.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.804674 8246 leader_election.cc:304] T 00000000000000000000000000000000 P 59d0ebb2088d4fafbfc147165d7544f8 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4c24d4441f6c47728564eedff9749c8f, 59d0ebb2088d4fafbfc147165d7544f8; no voters:
03:47:50.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.806262 8223 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.221.124" port: 35655 } attempt: 5
03:47:50.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.807557 8334 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 59d0ebb2088d4fafbfc147165d7544f8 [term 1 FOLLOWER]: Leader election won for term 1
03:47:50.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:50.811609 8248 tablet.cc:2378] T 00000000000000000000000000000000 P 59d0ebb2088d4fafbfc147165d7544f8: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:47:50.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.824126 8223 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5: Bootstrap starting.
03:47:50.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.828497 8311 leader_election.cc:290] T 00000000000000000000000000000000 P 59d0ebb2088d4fafbfc147165d7544f8 [CANDIDATE]: Term 1 election: Requested vote from peers 4c24d4441f6c47728564eedff9749c8f (127.7.221.126:43693), abe570d88d9e41929e679edcc29956c5 (127.7.221.125:33545)
03:47:50.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.838250 8223 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5: Neither blocks nor log segments found. Creating new log.
03:47:50.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.841204 8223 log.cc:826] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5: Log is configured to *not* fsync() on all Append() calls
03:47:50.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.842444 8334 raft_consensus.cc:695] T 00000000000000000000000000000000 P 59d0ebb2088d4fafbfc147165d7544f8 [term 1 LEADER]: Becoming Leader. State: Replica: 59d0ebb2088d4fafbfc147165d7544f8, State: Running, Role: LEADER
03:47:50.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.843648 8334 consensus_queue.cc:237] T 00000000000000000000000000000000 P 59d0ebb2088d4fafbfc147165d7544f8 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c24d4441f6c47728564eedff9749c8f" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 43693 } } peers { permanent_uuid: "abe570d88d9e41929e679edcc29956c5" member_type: VOTER last_known_addr { host: "127.7.221.125" port: 33545 } } peers { permanent_uuid: "59d0ebb2088d4fafbfc147165d7544f8" member_type: VOTER last_known_addr { host: "127.7.221.124" port: 35655 } }
03:47:50.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.849553 8223 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5: No bootstrap required, opened a new log
03:47:50.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.846894 8065 master_runner.cc:186] Error getting master registration for 127.7.221.125:33545: OK, instance_id { permanent_uuid: "abe570d88d9e41929e679edcc29956c5" instance_seqno: 1750823270096352 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:47:50.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.853709 8065 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:47:50.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:50.854501 8085 tablet.cc:2378] T 00000000000000000000000000000000 P 4c24d4441f6c47728564eedff9749c8f: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:47:50.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.858266 8223 raft_consensus.cc:357] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c24d4441f6c47728564eedff9749c8f" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 43693 } } peers { permanent_uuid: "abe570d88d9e41929e679edcc29956c5" member_type: VOTER last_known_addr { host: "127.7.221.125" port: 33545 } } peers { permanent_uuid: "59d0ebb2088d4fafbfc147165d7544f8" member_type: VOTER last_known_addr { host: "127.7.221.124" port: 35655 } }
03:47:50.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.858932 8223 raft_consensus.cc:383] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:47:50.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.859162 8223 raft_consensus.cc:738] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: abe570d88d9e41929e679edcc29956c5, State: Initialized, Role: FOLLOWER
03:47:50.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:50.859480 8338 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 4c24d4441f6c47728564eedff9749c8f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:47:50.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:50.859624 8338 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:47:50.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.860266 8223 consensus_queue.cc:260] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c24d4441f6c47728564eedff9749c8f" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 43693 } } peers { permanent_uuid: "abe570d88d9e41929e679edcc29956c5" member_type: VOTER last_known_addr { host: "127.7.221.125" port: 33545 } } peers { permanent_uuid: "59d0ebb2088d4fafbfc147165d7544f8" member_type: VOTER last_known_addr { host: "127.7.221.124" port: 35655 } }
03:47:50.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.862607 8342 sys_catalog.cc:455] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c24d4441f6c47728564eedff9749c8f" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 43693 } } peers { permanent_uuid: "abe570d88d9e41929e679edcc29956c5" member_type: VOTER last_known_addr { host: "127.7.221.125" port: 33545 } } peers { permanent_uuid: "59d0ebb2088d4fafbfc147165d7544f8" member_type: VOTER last_known_addr { host: "127.7.221.124" port: 35655 } } }
03:47:50.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.863477 8342 sys_catalog.cc:458] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 [sys.catalog]: This master's current role is: FOLLOWER
03:47:50.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.863968 8342 raft_consensus.cc:491] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:47:50.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.864431 8342 raft_consensus.cc:513] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c24d4441f6c47728564eedff9749c8f" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 43693 } } peers { permanent_uuid: "abe570d88d9e41929e679edcc29956c5" member_type: VOTER last_known_addr { host: "127.7.221.125" port: 33545 } } peers { permanent_uuid: "59d0ebb2088d4fafbfc147165d7544f8" member_type: VOTER last_known_addr { host: "127.7.221.124" port: 35655 } }
03:47:50.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.867763 8342 leader_election.cc:290] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4c24d4441f6c47728564eedff9749c8f (127.7.221.126:43693), 59d0ebb2088d4fafbfc147165d7544f8 (127.7.221.124:35655)
03:47:50.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.868494 8311 sys_catalog.cc:455] T 00000000000000000000000000000000 P 59d0ebb2088d4fafbfc147165d7544f8 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 59d0ebb2088d4fafbfc147165d7544f8. Latest consensus state: current_term: 1 leader_uuid: "59d0ebb2088d4fafbfc147165d7544f8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c24d4441f6c47728564eedff9749c8f" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 43693 } } peers { permanent_uuid: "abe570d88d9e41929e679edcc29956c5" member_type: VOTER last_known_addr { host: "127.7.221.125" port: 33545 } } peers { permanent_uuid: "59d0ebb2088d4fafbfc147165d7544f8" member_type: VOTER last_known_addr { host: "127.7.221.124" port: 35655 } } }
03:47:50.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.868840 8311 sys_catalog.cc:458] T 00000000000000000000000000000000 P 59d0ebb2088d4fafbfc147165d7544f8 [sys.catalog]: This master's current role is: LEADER
03:47:50.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.868659 8269 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "abe570d88d9e41929e679edcc29956c5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "59d0ebb2088d4fafbfc147165d7544f8" is_pre_election: true
03:47:50.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.871299 8223 sys_catalog.cc:564] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 [sys.catalog]: configured and running, proceeding with master startup.
03:47:50.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.871421 8186 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "59d0ebb2088d4fafbfc147165d7544f8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "abe570d88d9e41929e679edcc29956c5"
03:47:50.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.873507 8186 raft_consensus.cc:3058] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 [term 0 FOLLOWER]: Advancing to term 1
03:47:50.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.871857 8187 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "59d0ebb2088d4fafbfc147165d7544f8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "abe570d88d9e41929e679edcc29956c5" is_pre_election: true
03:47:50.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.879366 8186 raft_consensus.cc:2466] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 59d0ebb2088d4fafbfc147165d7544f8 in term 1.
03:47:50.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.887027 8344 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
03:47:50.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.892071 8344 catalog_manager.cc:1486] Initializing Kudu cluster ID...
03:47:50.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.893841 8106 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "abe570d88d9e41929e679edcc29956c5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4c24d4441f6c47728564eedff9749c8f" is_pre_election: true
03:47:50.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.894369 8106 raft_consensus.cc:2391] T 00000000000000000000000000000000 P 4c24d4441f6c47728564eedff9749c8f [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate abe570d88d9e41929e679edcc29956c5 in current term 1: Already voted for candidate 59d0ebb2088d4fafbfc147165d7544f8 in this term.
03:47:50.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.895767 8164 leader_election.cc:304] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: abe570d88d9e41929e679edcc29956c5; no voters: 4c24d4441f6c47728564eedff9749c8f, 59d0ebb2088d4fafbfc147165d7544f8
03:47:50.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.896684 8343 raft_consensus.cc:2747] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
03:47:50.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.918823 8186 raft_consensus.cc:1273] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 [term 1 FOLLOWER]: Refusing update from remote peer 59d0ebb2088d4fafbfc147165d7544f8: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:47:50.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.919978 8311 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 59d0ebb2088d4fafbfc147165d7544f8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "abe570d88d9e41929e679edcc29956c5" member_type: VOTER last_known_addr { host: "127.7.221.125" port: 33545 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:47:50.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.926147 8106 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 4c24d4441f6c47728564eedff9749c8f [term 1 FOLLOWER]: Refusing update from remote peer 59d0ebb2088d4fafbfc147165d7544f8: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:47:50.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.927361 8311 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 59d0ebb2088d4fafbfc147165d7544f8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c24d4441f6c47728564eedff9749c8f" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 43693 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:47:50.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.943941 8343 sys_catalog.cc:455] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 59d0ebb2088d4fafbfc147165d7544f8. Latest consensus state: current_term: 1 leader_uuid: "59d0ebb2088d4fafbfc147165d7544f8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c24d4441f6c47728564eedff9749c8f" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 43693 } } peers { permanent_uuid: "abe570d88d9e41929e679edcc29956c5" member_type: VOTER last_known_addr { host: "127.7.221.125" port: 33545 } } peers { permanent_uuid: "59d0ebb2088d4fafbfc147165d7544f8" member_type: VOTER last_known_addr { host: "127.7.221.124" port: 35655 } } }
03:47:50.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.944363 8343 sys_catalog.cc:458] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 [sys.catalog]: This master's current role is: FOLLOWER
03:47:50.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.947991 8348 mvcc.cc:204] Tried to move back new op lower bound from 7171372117667184640 to 7171372117387874304. Current Snapshot: MvccSnapshot[applied={T|T < 7171372117667184640}]
03:47:50.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.953320 8343 sys_catalog.cc:455] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "59d0ebb2088d4fafbfc147165d7544f8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c24d4441f6c47728564eedff9749c8f" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 43693 } } peers { permanent_uuid: "abe570d88d9e41929e679edcc29956c5" member_type: VOTER last_known_addr { host: "127.7.221.125" port: 33545 } } peers { permanent_uuid: "59d0ebb2088d4fafbfc147165d7544f8" member_type: VOTER last_known_addr { host: "127.7.221.124" port: 35655 } } }
03:47:50.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.953636 8343 sys_catalog.cc:458] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 [sys.catalog]: This master's current role is: FOLLOWER
03:47:50.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.955166 8351 mvcc.cc:204] Tried to move back new op lower bound from 7171372117667184640 to 7171372117387874304. Current Snapshot: MvccSnapshot[applied={T|T < 7171372117667184640}]
03:47:50.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.960968 8325 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4c24d4441f6c47728564eedff9749c8f [sys.catalog]: SysCatalogTable state changed. Reason: New leader 59d0ebb2088d4fafbfc147165d7544f8. Latest consensus state: current_term: 1 leader_uuid: "59d0ebb2088d4fafbfc147165d7544f8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c24d4441f6c47728564eedff9749c8f" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 43693 } } peers { permanent_uuid: "abe570d88d9e41929e679edcc29956c5" member_type: VOTER last_known_addr { host: "127.7.221.125" port: 33545 } } peers { permanent_uuid: "59d0ebb2088d4fafbfc147165d7544f8" member_type: VOTER last_known_addr { host: "127.7.221.124" port: 35655 } } }
03:47:50.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.961437 8325 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4c24d4441f6c47728564eedff9749c8f [sys.catalog]: This master's current role is: FOLLOWER
03:47:50.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.967419 8311 sys_catalog.cc:455] T 00000000000000000000000000000000 P 59d0ebb2088d4fafbfc147165d7544f8 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "59d0ebb2088d4fafbfc147165d7544f8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c24d4441f6c47728564eedff9749c8f" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 43693 } } peers { permanent_uuid: "abe570d88d9e41929e679edcc29956c5" member_type: VOTER last_known_addr { host: "127.7.221.125" port: 33545 } } peers { permanent_uuid: "59d0ebb2088d4fafbfc147165d7544f8" member_type: VOTER last_known_addr { host: "127.7.221.124" port: 35655 } } }
03:47:50.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.967797 8311 sys_catalog.cc:458] T 00000000000000000000000000000000 P 59d0ebb2088d4fafbfc147165d7544f8 [sys.catalog]: This master's current role is: LEADER
03:47:50.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.968261 8311 sys_catalog.cc:455] T 00000000000000000000000000000000 P 59d0ebb2088d4fafbfc147165d7544f8 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "59d0ebb2088d4fafbfc147165d7544f8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c24d4441f6c47728564eedff9749c8f" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 43693 } } peers { permanent_uuid: "abe570d88d9e41929e679edcc29956c5" member_type: VOTER last_known_addr { host: "127.7.221.125" port: 33545 } } peers { permanent_uuid: "59d0ebb2088d4fafbfc147165d7544f8" member_type: VOTER last_known_addr { host: "127.7.221.124" port: 35655 } } }
03:47:50.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.968577 8311 sys_catalog.cc:458] T 00000000000000000000000000000000 P 59d0ebb2088d4fafbfc147165d7544f8 [sys.catalog]: This master's current role is: LEADER
03:47:50.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.981370 8325 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4c24d4441f6c47728564eedff9749c8f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "59d0ebb2088d4fafbfc147165d7544f8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c24d4441f6c47728564eedff9749c8f" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 43693 } } peers { permanent_uuid: "abe570d88d9e41929e679edcc29956c5" member_type: VOTER last_known_addr { host: "127.7.221.125" port: 33545 } } peers { permanent_uuid: "59d0ebb2088d4fafbfc147165d7544f8" member_type: VOTER last_known_addr { host: "127.7.221.124" port: 35655 } } }
03:47:50.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.981688 8325 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4c24d4441f6c47728564eedff9749c8f [sys.catalog]: This master's current role is: FOLLOWER
03:47:50.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.981812 8344 catalog_manager.cc:1349] Generated new cluster ID: 01957b9e343a47d7bd313a67674c48c6
03:47:50.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:50.981935 8344 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
03:47:51.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:51.001029 8363 catalog_manager.cc:797] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
03:47:51.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:51.013024 8344 catalog_manager.cc:1372] Generated new certificate authority record
03:47:51.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:51.015337 8344 catalog_manager.cc:1506] Loading token signing keys...
03:47:51.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:51.038870 8344 catalog_manager.cc:5955] T 00000000000000000000000000000000 P 59d0ebb2088d4fafbfc147165d7544f8: Generated new TSK 0
03:47:51.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:51.040473 8344 catalog_manager.cc:1516] Initializing in-progress tserver states...
03:47:51.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:51.072605 8053 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskqbWxBn/build/asan/bin/kudu
03:47:51.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqbWxBn/build/asan/bin/kudu
03:47:51.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/ts-0/wal
03:47:51.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/ts-0/data
03:47:51.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:47:51.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:47:51.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:47:51.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:47:51.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:47:51.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:47:51.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/ts-0/logs
03:47:51.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/ts-0/data/info.pb
03:47:51.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:47:51.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:47:51.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:47:51.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:47:51.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:47:51.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:47:51.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:47:51.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:47:51.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:47:51.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:47:51.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.221.65:0
03:47:51.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.7.221.65
03:47:51.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.221.65
03:47:51.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:47:51.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655
03:47:51.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.221.84:35797
03:47:51.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:47:51.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:47:51.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:47:51.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
03:47:51.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
03:47:51.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
03:47:51.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:51.527479 8364 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:47:51.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:51.528231 8364 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:47:51.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:51.528445 8364 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:47:51.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:51.543254 8364 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:47:51.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:51.543467 8364 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
03:47:51.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:51.543553 8364 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
03:47:51.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:51.543628 8364 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
03:47:51.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:51.543768 8364 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.65
03:47:51.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:51.558208 8364 tablet_server_runner.cc:78] Tablet server non-default flags:
03:47:51.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:47:51.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.221.84:35797
03:47:51.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:47:51.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:47:51.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/ts-0/data
03:47:51.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/ts-0/wal
03:47:51.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:47:51.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:47:51.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.221.65:0
03:47:51.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:47:51.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:47:51.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:47:51.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/ts-0/data/info.pb
03:47:51.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.221.65
03:47:51.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:47:51.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
03:47:51.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
03:47:51.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
03:47:51.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655
03:47:51.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:47:51.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:47:51.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:47:51.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:47:51.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:47:51.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.7.221.65
03:47:51.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/ts-0/logs
03:47:51.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:47:51.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:47:51.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:47:51.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:47:51.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:47:51.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6d72d4a242076468501f3430b9a2cd050c634be2
03:47:51.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:47:51.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Jun 2025 03:18:58 UTC on 5fd53c4cbb9d
03:47:51.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6756
03:47:51.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
03:47:51.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:51.560577 8364 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:47:51.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:51.563056 8364 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:47:51.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:51.577211 8370 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:47:51.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:51.577242 8372 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:47:51.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:51.577283 8369 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:47:51.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:51.578240 8364 server_base.cc:1048] running on GCE node
03:47:51.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:51.578872 8364 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:47:51.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:51.579891 8364 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:47:51.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:51.581111 8364 hybrid_clock.cc:648] HybridClock initialized: now 1750823271581045 us; error 50 us; skew 500 ppm
03:47:51.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:51.581462 8364 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:47:51.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:51.584944 8364 webserver.cc:469] Webserver started at http://127.7.221.65:33465/ using document root <none> and password file <none>
03:47:51.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:51.585815 8364 fs_manager.cc:362] Metadata directory not provided
03:47:51.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:51.585966 8364 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:47:51.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:51.586331 8364 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
03:47:51.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:51.589355 8364 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/ts-0/data/instance:
03:47:51.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ef30ee403f0246569128f239c8802ffb"
03:47:51.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 03:47:51 on dist-test-slave-zzn2"
03:47:51.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:51.590529 8364 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/ts-0/wal/instance:
03:47:51.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ef30ee403f0246569128f239c8802ffb"
03:47:51.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 03:47:51 on dist-test-slave-zzn2"
03:47:51.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:51.597352 8364 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.006s sys 0.001s
03:47:51.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:51.601555 8379 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:47:51.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:51.603370 8364 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
03:47:51.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:51.603565 8364 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/ts-0/data,/tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/ts-0/wal
03:47:51.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ef30ee403f0246569128f239c8802ffb"
03:47:51.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 03:47:51 on dist-test-slave-zzn2"
03:47:51.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:51.603760 8364 fs_report.cc:389] FS layout report
03:47:51.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:47:51.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/ts-0/wal
03:47:51.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/ts-0/wal
03:47:51.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/ts-0/data/data
03:47:51.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:47:51.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:47:51.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:47:51.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:47:51.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:47:51.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:47:51.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:47:51.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:47:51.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:47:51.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:47:51.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:47:51.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:51.618834 8364 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:47:51.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:51.619792 8364 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:47:51.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:51.620077 8364 kserver.cc:163] Server-wide thread pool size limit: 3276
03:47:51.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:51.623472 8364 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:47:51.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:51.623611 8364 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:47:51.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:51.623744 8364 ts_tablet_manager.cc:610] Registered 0 tablets
03:47:51.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:51.623836 8364 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:47:51.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:51.697747 8364 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.65:34863
03:47:51.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:51.697803 8505 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.7.221.65:34863 every 8 connection(s)
03:47:51.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:51.699234 8364 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/ts-0/data/info.pb
03:47:51.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:51.702986 8507 heartbeater.cc:344] Connected to a master server at 127.7.221.126:43693
03:47:51.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:51.703459 8507 heartbeater.cc:461] Registering TS with master...
03:47:51.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:51.705190 8507 heartbeater.cc:507] Master 127.7.221.126:43693 requested a full tablet report, sending...
03:47:51.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:51.706655 8053 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskqbWxBn/build/asan/bin/kudu as pid 8364
03:47:51.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:51.706854 8053 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/ts-0/wal/instance
03:47:51.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:51.708170 8096 ts_manager.cc:194] Registered new tserver with Master: ef30ee403f0246569128f239c8802ffb (127.7.221.65:34863)
03:47:51.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:51.711421 8506 heartbeater.cc:344] Connected to a master server at 127.7.221.124:35655
03:47:51.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:51.711614 8506 heartbeater.cc:461] Registering TS with master...
03:47:51.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:51.711870 8053 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskqbWxBn/build/asan/bin/kudu
03:47:51.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqbWxBn/build/asan/bin/kudu
03:47:51.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/ts-1/wal
03:47:51.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/ts-1/data
03:47:51.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:47:51.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:47:51.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:47:51.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:47:51.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:47:51.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:47:51.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/ts-1/logs
03:47:51.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/ts-1/data/info.pb
03:47:51.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:47:51.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:47:51.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:47:51.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:47:51.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:47:51.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:47:51.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:47:51.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:47:51.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:47:51.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:47:51.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.221.66:0
03:47:51.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.7.221.66
03:47:51.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.221.66
03:47:51.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:47:51.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655
03:47:51.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.221.84:35797
03:47:51.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:47:51.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:47:51.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:47:51.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
03:47:51.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
03:47:51.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
03:47:51.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:51.712047 8506 heartbeater.cc:507] Master 127.7.221.124:35655 requested a full tablet report, sending...
03:47:51.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:51.713742 8259 ts_manager.cc:194] Registered new tserver with Master: ef30ee403f0246569128f239c8802ffb (127.7.221.65:34863)
03:47:51.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:51.715432 8259 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.65:33403
03:47:51.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:51.718274 8508 heartbeater.cc:344] Connected to a master server at 127.7.221.125:33545
03:47:51.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:51.718478 8508 heartbeater.cc:461] Registering TS with master...
03:47:51.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:51.718924 8508 heartbeater.cc:507] Master 127.7.221.125:33545 requested a full tablet report, sending...
03:47:51.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:51.720703 8176 ts_manager.cc:194] Registered new tserver with Master: ef30ee403f0246569128f239c8802ffb (127.7.221.65:34863)
03:47:51.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:51.861727 8338 catalog_manager.cc:1261] Loaded cluster ID: 01957b9e343a47d7bd313a67674c48c6
03:47:51.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:51.861835 8338 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 4c24d4441f6c47728564eedff9749c8f: loading cluster ID for follower catalog manager: success
03:47:51.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:51.864470 8338 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 4c24d4441f6c47728564eedff9749c8f: acquiring CA information for follower catalog manager: success
03:47:51.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:51.866614 8338 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 4c24d4441f6c47728564eedff9749c8f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:47:52.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.011204 8363 catalog_manager.cc:1261] Loaded cluster ID: 01957b9e343a47d7bd313a67674c48c6
03:47:52.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.011368 8363 catalog_manager.cc:1554] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5: loading cluster ID for follower catalog manager: success
03:47:52.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.015836 8363 catalog_manager.cc:1576] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5: acquiring CA information for follower catalog manager: success
03:47:52.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.019129 8363 catalog_manager.cc:1604] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:47:52.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:52.071851 8511 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:47:52.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:52.072255 8511 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:47:52.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:52.072443 8511 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:47:52.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:52.081339 8511 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:47:52.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:52.081566 8511 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
03:47:52.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:52.081665 8511 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
03:47:52.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:52.081729 8511 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
03:47:52.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:52.081888 8511 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.66
03:47:52.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.093505 8511 tablet_server_runner.cc:78] Tablet server non-default flags:
03:47:52.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:47:52.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.221.84:35797
03:47:52.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:47:52.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:47:52.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/ts-1/data
03:47:52.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/ts-1/wal
03:47:52.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:47:52.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:47:52.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.221.66:0
03:47:52.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:47:52.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:47:52.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:47:52.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/ts-1/data/info.pb
03:47:52.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.221.66
03:47:52.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:47:52.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
03:47:52.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
03:47:52.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
03:47:52.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655
03:47:52.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:47:52.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:47:52.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:47:52.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:47:52.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:47:52.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.7.221.66
03:47:52.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/ts-1/logs
03:47:52.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:47:52.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:47:52.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:47:52.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:47:52.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:47:52.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6d72d4a242076468501f3430b9a2cd050c634be2
03:47:52.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:47:52.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Jun 2025 03:18:58 UTC on 5fd53c4cbb9d
03:47:52.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6756
03:47:52.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
03:47:52.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.095676 8511 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:47:52.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.097920 8511 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:47:52.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:52.116717 8518 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:47:52.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:52.117020 8517 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:47:52.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:52.119277 8520 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:47:52.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.120507 8511 server_base.cc:1048] running on GCE node
03:47:52.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.121495 8511 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:47:52.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.122983 8511 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:47:52.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.124320 8511 hybrid_clock.cc:648] HybridClock initialized: now 1750823272124209 us; error 72 us; skew 500 ppm
03:47:52.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.124807 8511 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:47:52.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.128572 8511 webserver.cc:469] Webserver started at http://127.7.221.66:45983/ using document root <none> and password file <none>
03:47:52.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.129705 8511 fs_manager.cc:362] Metadata directory not provided
03:47:52.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.129879 8511 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:47:52.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.130308 8511 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
03:47:52.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.134311 8511 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/ts-1/data/instance:
03:47:52.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4a67bc06701742edb8d77170ed4da410"
03:47:52.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 03:47:52 on dist-test-slave-zzn2"
03:47:52.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.135607 8511 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/ts-1/wal/instance:
03:47:52.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4a67bc06701742edb8d77170ed4da410"
03:47:52.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 03:47:52 on dist-test-slave-zzn2"
03:47:52.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.144166 8511 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.010s sys 0.000s
03:47:52.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.149776 8526 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:47:52.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.151710 8511 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
03:47:52.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.151906 8511 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/ts-1/data,/tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/ts-1/wal
03:47:52.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4a67bc06701742edb8d77170ed4da410"
03:47:52.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 03:47:52 on dist-test-slave-zzn2"
03:47:52.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.152101 8511 fs_report.cc:389] FS layout report
03:47:52.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:47:52.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/ts-1/wal
03:47:52.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/ts-1/wal
03:47:52.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/ts-1/data/data
03:47:52.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:47:52.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:47:52.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:47:52.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:47:52.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:47:52.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:47:52.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:47:52.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:47:52.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:47:52.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:47:52.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:47:52.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.177803 8511 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:47:52.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.178900 8511 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:47:52.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.179190 8511 kserver.cc:163] Server-wide thread pool size limit: 3276
03:47:52.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.182698 8511 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:47:52.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.182862 8511 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:47:52.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.182986 8511 ts_tablet_manager.cc:610] Registered 0 tablets
03:47:52.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.183069 8511 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:47:52.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.283093 8511 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.66:45859
03:47:52.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.283113 8652 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.7.221.66:45859 every 8 connection(s)
03:47:52.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.285070 8511 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/ts-1/data/info.pb
03:47:52.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.287145 8053 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskqbWxBn/build/asan/bin/kudu as pid 8511
03:47:52.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.287346 8053 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/ts-1/wal/instance
03:47:52.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.293900 8053 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskqbWxBn/build/asan/bin/kudu
03:47:52.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqbWxBn/build/asan/bin/kudu
03:47:52.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/ts-2/wal
03:47:52.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/ts-2/data
03:47:52.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:47:52.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:47:52.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:47:52.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:47:52.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:47:52.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:47:52.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/ts-2/logs
03:47:52.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/ts-2/data/info.pb
03:47:52.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:47:52.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:47:52.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:47:52.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:47:52.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:47:52.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:47:52.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:47:52.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:47:52.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:47:52.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:47:52.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.221.67:0
03:47:52.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.7.221.67
03:47:52.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.221.67
03:47:52.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:47:52.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655
03:47:52.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.221.84:35797
03:47:52.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:47:52.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:47:52.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:47:52.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
03:47:52.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
03:47:52.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
03:47:52.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.302014 8654 heartbeater.cc:344] Connected to a master server at 127.7.221.126:43693
03:47:52.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.302932 8654 heartbeater.cc:461] Registering TS with master...
03:47:52.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.304781 8654 heartbeater.cc:507] Master 127.7.221.126:43693 requested a full tablet report, sending...
03:47:52.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.309110 8096 ts_manager.cc:194] Registered new tserver with Master: 4a67bc06701742edb8d77170ed4da410 (127.7.221.66:45859)
03:47:52.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.311444 8653 heartbeater.cc:344] Connected to a master server at 127.7.221.124:35655
03:47:52.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.311642 8653 heartbeater.cc:461] Registering TS with master...
03:47:52.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.312299 8653 heartbeater.cc:507] Master 127.7.221.124:35655 requested a full tablet report, sending...
03:47:52.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.313772 8259 ts_manager.cc:194] Registered new tserver with Master: 4a67bc06701742edb8d77170ed4da410 (127.7.221.66:45859)
03:47:52.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.315367 8259 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.66:54329
03:47:52.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.332479 8655 heartbeater.cc:344] Connected to a master server at 127.7.221.125:33545
03:47:52.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.332716 8655 heartbeater.cc:461] Registering TS with master...
03:47:52.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.333175 8655 heartbeater.cc:507] Master 127.7.221.125:33545 requested a full tablet report, sending...
03:47:52.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.334352 8176 ts_manager.cc:194] Registered new tserver with Master: 4a67bc06701742edb8d77170ed4da410 (127.7.221.66:45859)
03:47:52.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:52.710435 8656 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:47:52.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:52.711086 8656 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:47:52.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:52.711221 8656 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:47:52.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.719350 8506 heartbeater.cc:499] Master 127.7.221.124:35655 was elected leader, sending a full tablet report...
03:47:52.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:52.719530 8656 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:47:52.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:52.719723 8656 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
03:47:52.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:52.719807 8656 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
03:47:52.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:52.719875 8656 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
03:47:52.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:52.720024 8656 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.221.67
03:47:52.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.730759 8656 tablet_server_runner.cc:78] Tablet server non-default flags:
03:47:52.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:47:52.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.221.84:35797
03:47:52.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:47:52.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:47:52.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/ts-2/data
03:47:52.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/ts-2/wal
03:47:52.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:47:52.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:47:52.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.221.67:0
03:47:52.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:47:52.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:47:52.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:47:52.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/ts-2/data/info.pb
03:47:52.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.221.67
03:47:52.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:47:52.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
03:47:52.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
03:47:52.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
03:47:52.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655
03:47:52.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:47:52.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:47:52.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:47:52.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:47:52.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:47:52.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.7.221.67
03:47:52.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/ts-2/logs
03:47:52.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:47:52.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:47:52.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:47:52.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:47:52.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:47:52.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6d72d4a242076468501f3430b9a2cd050c634be2
03:47:52.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:47:52.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Jun 2025 03:18:58 UTC on 5fd53c4cbb9d
03:47:52.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6756
03:47:52.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
03:47:52.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.732692 8656 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:47:52.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.734808 8656 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:47:52.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:52.748397 8664 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:47:52.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:52.748391 8666 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:47:52.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:52.749357 8663 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:47:52.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.750249 8656 server_base.cc:1048] running on GCE node
03:47:52.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.750990 8656 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:47:52.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.752038 8656 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:47:52.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.753293 8656 hybrid_clock.cc:648] HybridClock initialized: now 1750823272753228 us; error 35 us; skew 500 ppm
03:47:52.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.753690 8656 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:47:52.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.756392 8656 webserver.cc:469] Webserver started at http://127.7.221.67:44193/ using document root <none> and password file <none>
03:47:52.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.757332 8656 fs_manager.cc:362] Metadata directory not provided
03:47:52.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.757490 8656 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:47:52.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.758044 8656 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
03:47:52.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.760989 8656 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/ts-2/data/instance:
03:47:52.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "79d1b3f1a73945fe9d116354bf26157f"
03:47:52.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 03:47:52 on dist-test-slave-zzn2"
03:47:52.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.761992 8656 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/ts-2/wal/instance:
03:47:52.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "79d1b3f1a73945fe9d116354bf26157f"
03:47:52.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 03:47:52 on dist-test-slave-zzn2"
03:47:52.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.768350 8656 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.006s sys 0.000s
03:47:52.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.772455 8672 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:47:52.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.774197 8656 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
03:47:52.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.774426 8656 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/ts-2/data,/tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/ts-2/wal
03:47:52.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "79d1b3f1a73945fe9d116354bf26157f"
03:47:52.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 03:47:52 on dist-test-slave-zzn2"
03:47:52.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.774693 8656 fs_report.cc:389] FS layout report
03:47:52.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:47:52.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/ts-2/wal
03:47:52.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/ts-2/wal
03:47:52.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/ts-2/data/data
03:47:52.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:47:52.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:47:52.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:47:52.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:47:52.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:47:52.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:47:52.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:47:52.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:47:52.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:47:52.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:47:52.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:47:52.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.789569 8656 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:47:52.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.790522 8656 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:47:52.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.790844 8656 kserver.cc:163] Server-wide thread pool size limit: 3276
03:47:52.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.794064 8656 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:47:52.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.794229 8656 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:47:52.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.794363 8656 ts_tablet_manager.cc:610] Registered 0 tablets
03:47:52.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.794463 8656 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:47:52.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.871652 8656 rpc_server.cc:307] RPC server started. Bound to: 127.7.221.67:42171
03:47:52.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.871716 8797 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.7.221.67:42171 every 8 connection(s)
03:47:52.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.873560 8656 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/ts-2/data/info.pb
03:47:52.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.874107 8053 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskqbWxBn/build/asan/bin/kudu as pid 8656
03:47:52.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.874341 8053 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskqbWxBn/test-tmp/mini-kudu-cluster6926944562489489001/ts-2/wal/instance
03:47:52.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.876955 8799 heartbeater.cc:344] Connected to a master server at 127.7.221.126:43693
03:47:52.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.877460 8799 heartbeater.cc:461] Registering TS with master...
03:47:52.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.878842 8799 heartbeater.cc:507] Master 127.7.221.126:43693 requested a full tablet report, sending...
03:47:52.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.881258 8096 ts_manager.cc:194] Registered new tserver with Master: 79d1b3f1a73945fe9d116354bf26157f (127.7.221.67:42171)
03:47:52.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.887796 8800 heartbeater.cc:344] Connected to a master server at 127.7.221.125:33545
03:47:52.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.888018 8800 heartbeater.cc:461] Registering TS with master...
03:47:52.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.888721 8800 heartbeater.cc:507] Master 127.7.221.125:33545 requested a full tablet report, sending...
03:47:52.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.889940 8176 ts_manager.cc:194] Registered new tserver with Master: 79d1b3f1a73945fe9d116354bf26157f (127.7.221.67:42171)
03:47:52.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.892992 8798 heartbeater.cc:344] Connected to a master server at 127.7.221.124:35655
03:47:52.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.893143 8798 heartbeater.cc:461] Registering TS with master...
03:47:52.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.893553 8798 heartbeater.cc:507] Master 127.7.221.124:35655 requested a full tablet report, sending...
03:47:52.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.894598 8259 ts_manager.cc:194] Registered new tserver with Master: 79d1b3f1a73945fe9d116354bf26157f (127.7.221.67:42171)
03:47:52.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.895432 8259 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.221.67:49321
03:47:52.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.901019 8053 external_mini_cluster.cc:934] 3 TS(s) registered with all masters
03:47:52.903 [DEBUG - main] (MiniKuduCluster.java:195) Response:
03:47:52.903 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
03:47:52.904 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.7.221.126"
port: 43693
}
bound_http_address {
host: "127.7.221.126"
port: 38867
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.7.221.125"
port: 33545
}
bound_http_address {
host: "127.7.221.125"
port: 43235
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.7.221.124"
port: 35655
}
bound_http_address {
host: "127.7.221.124"
port: 42033
}
}
}
03:47:52.904 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
03:47:52.905 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.7.221.65"
port: 34863
}
bound_http_address {
host: "127.7.221.65"
port: 33465
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.7.221.66"
port: 45859
}
bound_http_address {
host: "127.7.221.66"
port: 45983
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.7.221.67"
port: 42171
}
bound_http_address {
host: "127.7.221.67"
port: 44193
}
}
}
03:47:52.905 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
03:47:52.907 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
03:47:52.908 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.7.221.126:43693(127.7.221.126:43693)] connecting to peer
03:47:52.910 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.7.221.125:33545(127.7.221.125:33545)] connecting to peer
03:47:52.911 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.7.221.124:35655(127.7.221.124:35655)] connecting to peer
03:47:52.910 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.7.221.126:43693(127.7.221.126:43693)] Successfully connected to peer
03:47:52.912 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.7.221.125:33545(127.7.221.125:33545)] Successfully connected to peer
03:47:52.915 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.7.221.126
03:47:52.915 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.7.221.125
03:47:52.920 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.7.221.124:35655(127.7.221.124:35655)] Successfully connected to peer
03:47:52.930 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.7.221.124
03:47:52.930 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x6cafb5e2, L:/127.0.0.1:34660 - R:/127.7.221.125:33545] using SASL/PLAIN
03:47:52.934 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x72434226, L:/127.0.0.1:53292 - R:/127.7.221.126:43693] using SASL/PLAIN
03:47:52.943 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x9fa36392, L:/127.0.0.1:54700 - R:/127.7.221.124:35655] using SASL/PLAIN
03:47:52.946 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
03:47:52.946 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=38), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.126:43693, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.125:33545, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.124:35655, [23ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.125:33545: OK, [28ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.126:43693: OK, [37ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.124:35655: OK, deferred=null) after lookup
03:47:52.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:52.948799 8259 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:54700:
03:47:52.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_manager_fallback"
03:47:52.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
03:47:52.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:47:52.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
03:47:52.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
03:47:52.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
03:47:52.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:47:52.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:47:52.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:47:52.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
03:47:52.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:47:52.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:47:52.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column1_i"
03:47:52.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
03:47:52.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
03:47:52.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:47:52.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:47:52.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:47:52.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
03:47:52.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:47:52.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:47:52.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column2_i"
03:47:52.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
03:47:52.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
03:47:52.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:47:52.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:47:52.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:47:52.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
03:47:52.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:47:52.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:47:52.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column3_s"
03:47:52.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
03:47:52.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
03:47:52.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
03:47:52.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: DICT_ENCODING
03:47:52.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: LZ4
03:47:52.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 4096
03:47:52.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:47:52.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
03:47:52.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:47:52.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:47:52.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column4_b"
03:47:52.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: BOOL
03:47:52.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
03:47:52.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:47:52.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:47:52.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:47:52.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
03:47:52.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:47:52.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:47:52.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
03:47:52.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) hash_schema {
03:47:52.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:47:52.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
03:47:52.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:47:52.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_buckets: 2
03:47:52.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) seed: 0
03:47:52.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:47:52.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:47:52.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:52.951534 8259 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table txn_manager_fallback in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
03:47:52.979 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 1f8120a6dc0743938fb8db3498d07cae
03:47:53.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.024545 8440 tablet_service.cc:1468] Processing CreateTablet for tablet f35afbde3ffe42e9aa2a447b50aed40c (DEFAULT_TABLE table=txn_manager_fallback [id=1f8120a6dc0743938fb8db3498d07cae]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
03:47:53.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.026034 8727 tablet_service.cc:1468] Processing CreateTablet for tablet f35afbde3ffe42e9aa2a447b50aed40c (DEFAULT_TABLE table=txn_manager_fallback [id=1f8120a6dc0743938fb8db3498d07cae]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
03:47:53.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.025890 8432 tablet_service.cc:1468] Processing CreateTablet for tablet 729fc2d921ee42a0b407222da665b01c (DEFAULT_TABLE table=txn_manager_fallback [id=1f8120a6dc0743938fb8db3498d07cae]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
03:47:53.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.027628 8727 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f35afbde3ffe42e9aa2a447b50aed40c. 1 dirs total, 0 dirs full, 0 dirs failed
03:47:53.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.027626 8440 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f35afbde3ffe42e9aa2a447b50aed40c. 1 dirs total, 0 dirs full, 0 dirs failed
03:47:53.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.028299 8432 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 729fc2d921ee42a0b407222da665b01c. 1 dirs total, 0 dirs full, 0 dirs failed
03:47:53.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.028409 8580 tablet_service.cc:1468] Processing CreateTablet for tablet 729fc2d921ee42a0b407222da665b01c (DEFAULT_TABLE table=txn_manager_fallback [id=1f8120a6dc0743938fb8db3498d07cae]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
03:47:53.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.030015 8580 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 729fc2d921ee42a0b407222da665b01c. 1 dirs total, 0 dirs full, 0 dirs failed
03:47:53.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.033241 8581 tablet_service.cc:1468] Processing CreateTablet for tablet f35afbde3ffe42e9aa2a447b50aed40c (DEFAULT_TABLE table=txn_manager_fallback [id=1f8120a6dc0743938fb8db3498d07cae]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
03:47:53.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.033809 8581 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f35afbde3ffe42e9aa2a447b50aed40c. 1 dirs total, 0 dirs full, 0 dirs failed
03:47:53.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.035461 8725 tablet_service.cc:1468] Processing CreateTablet for tablet 729fc2d921ee42a0b407222da665b01c (DEFAULT_TABLE table=txn_manager_fallback [id=1f8120a6dc0743938fb8db3498d07cae]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
03:47:53.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.036063 8725 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 729fc2d921ee42a0b407222da665b01c. 1 dirs total, 0 dirs full, 0 dirs failed
03:47:53.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.048985 8824 tablet_bootstrap.cc:492] T f35afbde3ffe42e9aa2a447b50aed40c P 79d1b3f1a73945fe9d116354bf26157f: Bootstrap starting.
03:47:53.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.054055 8824 tablet_bootstrap.cc:654] T f35afbde3ffe42e9aa2a447b50aed40c P 79d1b3f1a73945fe9d116354bf26157f: Neither blocks nor log segments found. Creating new log.
03:47:53.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.055650 8824 log.cc:826] T f35afbde3ffe42e9aa2a447b50aed40c P 79d1b3f1a73945fe9d116354bf26157f: Log is configured to *not* fsync() on all Append() calls
03:47:53.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.061902 8826 tablet_bootstrap.cc:492] T f35afbde3ffe42e9aa2a447b50aed40c P 4a67bc06701742edb8d77170ed4da410: Bootstrap starting.
03:47:53.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.064677 8824 tablet_bootstrap.cc:492] T f35afbde3ffe42e9aa2a447b50aed40c P 79d1b3f1a73945fe9d116354bf26157f: No bootstrap required, opened a new log
03:47:53.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.065099 8824 ts_tablet_manager.cc:1397] T f35afbde3ffe42e9aa2a447b50aed40c P 79d1b3f1a73945fe9d116354bf26157f: Time spent bootstrapping tablet: real 0.016s user 0.009s sys 0.006s
03:47:53.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.065713 8825 tablet_bootstrap.cc:492] T 729fc2d921ee42a0b407222da665b01c P ef30ee403f0246569128f239c8802ffb: Bootstrap starting.
03:47:53.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.066394 8826 tablet_bootstrap.cc:654] T f35afbde3ffe42e9aa2a447b50aed40c P 4a67bc06701742edb8d77170ed4da410: Neither blocks nor log segments found. Creating new log.
03:47:53.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.068034 8826 log.cc:826] T f35afbde3ffe42e9aa2a447b50aed40c P 4a67bc06701742edb8d77170ed4da410: Log is configured to *not* fsync() on all Append() calls
03:47:53.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.070125 8825 tablet_bootstrap.cc:654] T 729fc2d921ee42a0b407222da665b01c P ef30ee403f0246569128f239c8802ffb: Neither blocks nor log segments found. Creating new log.
03:47:53.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.071664 8825 log.cc:826] T 729fc2d921ee42a0b407222da665b01c P ef30ee403f0246569128f239c8802ffb: Log is configured to *not* fsync() on all Append() calls
03:47:53.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.071900 8826 tablet_bootstrap.cc:492] T f35afbde3ffe42e9aa2a447b50aed40c P 4a67bc06701742edb8d77170ed4da410: No bootstrap required, opened a new log
03:47:53.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.072227 8826 ts_tablet_manager.cc:1397] T f35afbde3ffe42e9aa2a447b50aed40c P 4a67bc06701742edb8d77170ed4da410: Time spent bootstrapping tablet: real 0.011s user 0.006s sys 0.002s
03:47:53.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.072347 8824 raft_consensus.cc:357] T f35afbde3ffe42e9aa2a447b50aed40c P 79d1b3f1a73945fe9d116354bf26157f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef30ee403f0246569128f239c8802ffb" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34863 } } peers { permanent_uuid: "79d1b3f1a73945fe9d116354bf26157f" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42171 } } peers { permanent_uuid: "4a67bc06701742edb8d77170ed4da410" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 45859 } }
03:47:53.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.072835 8824 raft_consensus.cc:383] T f35afbde3ffe42e9aa2a447b50aed40c P 79d1b3f1a73945fe9d116354bf26157f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:47:53.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.072975 8824 raft_consensus.cc:738] T f35afbde3ffe42e9aa2a447b50aed40c P 79d1b3f1a73945fe9d116354bf26157f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 79d1b3f1a73945fe9d116354bf26157f, State: Initialized, Role: FOLLOWER
03:47:53.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.073858 8824 consensus_queue.cc:260] T f35afbde3ffe42e9aa2a447b50aed40c P 79d1b3f1a73945fe9d116354bf26157f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef30ee403f0246569128f239c8802ffb" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34863 } } peers { permanent_uuid: "79d1b3f1a73945fe9d116354bf26157f" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42171 } } peers { permanent_uuid: "4a67bc06701742edb8d77170ed4da410" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 45859 } }
03:47:53.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.075273 8798 heartbeater.cc:499] Master 127.7.221.124:35655 was elected leader, sending a full tablet report...
03:47:53.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.075976 8824 ts_tablet_manager.cc:1428] T f35afbde3ffe42e9aa2a447b50aed40c P 79d1b3f1a73945fe9d116354bf26157f: Time spent starting tablet: real 0.011s user 0.009s sys 0.001s
03:47:53.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.077401 8825 tablet_bootstrap.cc:492] T 729fc2d921ee42a0b407222da665b01c P ef30ee403f0246569128f239c8802ffb: No bootstrap required, opened a new log
03:47:53.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.077770 8825 ts_tablet_manager.cc:1397] T 729fc2d921ee42a0b407222da665b01c P ef30ee403f0246569128f239c8802ffb: Time spent bootstrapping tablet: real 0.012s user 0.010s sys 0.000s
03:47:53.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.079833 8824 tablet_bootstrap.cc:492] T 729fc2d921ee42a0b407222da665b01c P 79d1b3f1a73945fe9d116354bf26157f: Bootstrap starting.
03:47:53.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.080269 8826 raft_consensus.cc:357] T f35afbde3ffe42e9aa2a447b50aed40c P 4a67bc06701742edb8d77170ed4da410 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef30ee403f0246569128f239c8802ffb" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34863 } } peers { permanent_uuid: "79d1b3f1a73945fe9d116354bf26157f" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42171 } } peers { permanent_uuid: "4a67bc06701742edb8d77170ed4da410" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 45859 } }
03:47:53.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.080812 8826 raft_consensus.cc:383] T f35afbde3ffe42e9aa2a447b50aed40c P 4a67bc06701742edb8d77170ed4da410 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:47:53.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.081072 8826 raft_consensus.cc:738] T f35afbde3ffe42e9aa2a447b50aed40c P 4a67bc06701742edb8d77170ed4da410 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4a67bc06701742edb8d77170ed4da410, State: Initialized, Role: FOLLOWER
03:47:53.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.081976 8826 consensus_queue.cc:260] T f35afbde3ffe42e9aa2a447b50aed40c P 4a67bc06701742edb8d77170ed4da410 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef30ee403f0246569128f239c8802ffb" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34863 } } peers { permanent_uuid: "79d1b3f1a73945fe9d116354bf26157f" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42171 } } peers { permanent_uuid: "4a67bc06701742edb8d77170ed4da410" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 45859 } }
03:47:53.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.082738 8824 tablet_bootstrap.cc:654] T 729fc2d921ee42a0b407222da665b01c P 79d1b3f1a73945fe9d116354bf26157f: Neither blocks nor log segments found. Creating new log.
03:47:53.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.086459 8825 raft_consensus.cc:357] T 729fc2d921ee42a0b407222da665b01c P ef30ee403f0246569128f239c8802ffb [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef30ee403f0246569128f239c8802ffb" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34863 } } peers { permanent_uuid: "4a67bc06701742edb8d77170ed4da410" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 45859 } } peers { permanent_uuid: "79d1b3f1a73945fe9d116354bf26157f" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42171 } }
03:47:53.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.087124 8825 raft_consensus.cc:383] T 729fc2d921ee42a0b407222da665b01c P ef30ee403f0246569128f239c8802ffb [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:47:53.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.087251 8825 raft_consensus.cc:738] T 729fc2d921ee42a0b407222da665b01c P ef30ee403f0246569128f239c8802ffb [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ef30ee403f0246569128f239c8802ffb, State: Initialized, Role: FOLLOWER
03:47:53.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.088311 8825 consensus_queue.cc:260] T 729fc2d921ee42a0b407222da665b01c P ef30ee403f0246569128f239c8802ffb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef30ee403f0246569128f239c8802ffb" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34863 } } peers { permanent_uuid: "4a67bc06701742edb8d77170ed4da410" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 45859 } } peers { permanent_uuid: "79d1b3f1a73945fe9d116354bf26157f" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42171 } }
03:47:53.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.097220 8653 heartbeater.cc:499] Master 127.7.221.124:35655 was elected leader, sending a full tablet report...
03:47:53.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.099720 8826 ts_tablet_manager.cc:1428] T f35afbde3ffe42e9aa2a447b50aed40c P 4a67bc06701742edb8d77170ed4da410: Time spent starting tablet: real 0.027s user 0.011s sys 0.012s
03:47:53.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.100284 8826 tablet_bootstrap.cc:492] T 729fc2d921ee42a0b407222da665b01c P 4a67bc06701742edb8d77170ed4da410: Bootstrap starting.
03:47:53.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.102463 8826 tablet_bootstrap.cc:654] T 729fc2d921ee42a0b407222da665b01c P 4a67bc06701742edb8d77170ed4da410: Neither blocks nor log segments found. Creating new log.
03:47:53.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.106164 8824 tablet_bootstrap.cc:492] T 729fc2d921ee42a0b407222da665b01c P 79d1b3f1a73945fe9d116354bf26157f: No bootstrap required, opened a new log
03:47:53.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.106611 8824 ts_tablet_manager.cc:1397] T 729fc2d921ee42a0b407222da665b01c P 79d1b3f1a73945fe9d116354bf26157f: Time spent bootstrapping tablet: real 0.027s user 0.004s sys 0.012s
03:47:53.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.107592 8824 raft_consensus.cc:357] T 729fc2d921ee42a0b407222da665b01c P 79d1b3f1a73945fe9d116354bf26157f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef30ee403f0246569128f239c8802ffb" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34863 } } peers { permanent_uuid: "4a67bc06701742edb8d77170ed4da410" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 45859 } } peers { permanent_uuid: "79d1b3f1a73945fe9d116354bf26157f" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42171 } }
03:47:53.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.108451 8824 raft_consensus.cc:383] T 729fc2d921ee42a0b407222da665b01c P 79d1b3f1a73945fe9d116354bf26157f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:47:53.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.108717 8824 raft_consensus.cc:738] T 729fc2d921ee42a0b407222da665b01c P 79d1b3f1a73945fe9d116354bf26157f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 79d1b3f1a73945fe9d116354bf26157f, State: Initialized, Role: FOLLOWER
03:47:53.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.109114 8824 consensus_queue.cc:260] T 729fc2d921ee42a0b407222da665b01c P 79d1b3f1a73945fe9d116354bf26157f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef30ee403f0246569128f239c8802ffb" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34863 } } peers { permanent_uuid: "4a67bc06701742edb8d77170ed4da410" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 45859 } } peers { permanent_uuid: "79d1b3f1a73945fe9d116354bf26157f" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42171 } }
03:47:53.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.112924 8824 ts_tablet_manager.cc:1428] T 729fc2d921ee42a0b407222da665b01c P 79d1b3f1a73945fe9d116354bf26157f: Time spent starting tablet: real 0.006s user 0.002s sys 0.000s
03:47:53.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.119247 8825 ts_tablet_manager.cc:1428] T 729fc2d921ee42a0b407222da665b01c P ef30ee403f0246569128f239c8802ffb: Time spent starting tablet: real 0.041s user 0.012s sys 0.011s
03:47:53.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.123322 8825 tablet_bootstrap.cc:492] T f35afbde3ffe42e9aa2a447b50aed40c P ef30ee403f0246569128f239c8802ffb: Bootstrap starting.
03:47:53.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.125622 8825 tablet_bootstrap.cc:654] T f35afbde3ffe42e9aa2a447b50aed40c P ef30ee403f0246569128f239c8802ffb: Neither blocks nor log segments found. Creating new log.
03:47:53.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.128098 8826 tablet_bootstrap.cc:492] T 729fc2d921ee42a0b407222da665b01c P 4a67bc06701742edb8d77170ed4da410: No bootstrap required, opened a new log
03:47:53.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.128330 8826 ts_tablet_manager.cc:1397] T 729fc2d921ee42a0b407222da665b01c P 4a67bc06701742edb8d77170ed4da410: Time spent bootstrapping tablet: real 0.028s user 0.019s sys 0.006s
03:47:53.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.129331 8825 tablet_bootstrap.cc:492] T f35afbde3ffe42e9aa2a447b50aed40c P ef30ee403f0246569128f239c8802ffb: No bootstrap required, opened a new log
03:47:53.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.129242 8826 raft_consensus.cc:357] T 729fc2d921ee42a0b407222da665b01c P 4a67bc06701742edb8d77170ed4da410 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef30ee403f0246569128f239c8802ffb" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34863 } } peers { permanent_uuid: "4a67bc06701742edb8d77170ed4da410" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 45859 } } peers { permanent_uuid: "79d1b3f1a73945fe9d116354bf26157f" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42171 } }
03:47:53.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.129477 8826 raft_consensus.cc:383] T 729fc2d921ee42a0b407222da665b01c P 4a67bc06701742edb8d77170ed4da410 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:47:53.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.129542 8825 ts_tablet_manager.cc:1397] T f35afbde3ffe42e9aa2a447b50aed40c P ef30ee403f0246569128f239c8802ffb: Time spent bootstrapping tablet: real 0.006s user 0.000s sys 0.003s
03:47:53.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.129547 8826 raft_consensus.cc:738] T 729fc2d921ee42a0b407222da665b01c P 4a67bc06701742edb8d77170ed4da410 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4a67bc06701742edb8d77170ed4da410, State: Initialized, Role: FOLLOWER
03:47:53.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.129923 8826 consensus_queue.cc:260] T 729fc2d921ee42a0b407222da665b01c P 4a67bc06701742edb8d77170ed4da410 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef30ee403f0246569128f239c8802ffb" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34863 } } peers { permanent_uuid: "4a67bc06701742edb8d77170ed4da410" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 45859 } } peers { permanent_uuid: "79d1b3f1a73945fe9d116354bf26157f" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42171 } }
03:47:53.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.130553 8826 ts_tablet_manager.cc:1428] T 729fc2d921ee42a0b407222da665b01c P 4a67bc06701742edb8d77170ed4da410: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
03:47:53.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.130329 8825 raft_consensus.cc:357] T f35afbde3ffe42e9aa2a447b50aed40c P ef30ee403f0246569128f239c8802ffb [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef30ee403f0246569128f239c8802ffb" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34863 } } peers { permanent_uuid: "79d1b3f1a73945fe9d116354bf26157f" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42171 } } peers { permanent_uuid: "4a67bc06701742edb8d77170ed4da410" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 45859 } }
03:47:53.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.131215 8825 raft_consensus.cc:383] T f35afbde3ffe42e9aa2a447b50aed40c P ef30ee403f0246569128f239c8802ffb [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:47:53.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.131605 8825 raft_consensus.cc:738] T f35afbde3ffe42e9aa2a447b50aed40c P ef30ee403f0246569128f239c8802ffb [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ef30ee403f0246569128f239c8802ffb, State: Initialized, Role: FOLLOWER
03:47:53.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.131969 8825 consensus_queue.cc:260] T f35afbde3ffe42e9aa2a447b50aed40c P ef30ee403f0246569128f239c8802ffb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef30ee403f0246569128f239c8802ffb" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34863 } } peers { permanent_uuid: "79d1b3f1a73945fe9d116354bf26157f" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42171 } } peers { permanent_uuid: "4a67bc06701742edb8d77170ed4da410" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 45859 } }
03:47:53.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.134233 8825 ts_tablet_manager.cc:1428] T f35afbde3ffe42e9aa2a447b50aed40c P ef30ee403f0246569128f239c8802ffb: Time spent starting tablet: real 0.005s user 0.001s sys 0.001s
03:47:53.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:53.134661 8801 tablet.cc:2378] T 729fc2d921ee42a0b407222da665b01c P 79d1b3f1a73945fe9d116354bf26157f: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:47:53.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:53.203285 8509 tablet.cc:2378] T 729fc2d921ee42a0b407222da665b01c P ef30ee403f0246569128f239c8802ffb: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:47:53.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.238461 8833 raft_consensus.cc:491] T f35afbde3ffe42e9aa2a447b50aed40c P ef30ee403f0246569128f239c8802ffb [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:47:53.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.238752 8833 raft_consensus.cc:513] T f35afbde3ffe42e9aa2a447b50aed40c P ef30ee403f0246569128f239c8802ffb [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef30ee403f0246569128f239c8802ffb" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34863 } } peers { permanent_uuid: "79d1b3f1a73945fe9d116354bf26157f" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42171 } } peers { permanent_uuid: "4a67bc06701742edb8d77170ed4da410" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 45859 } }
03:47:53.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.240741 8833 leader_election.cc:290] T f35afbde3ffe42e9aa2a447b50aed40c P ef30ee403f0246569128f239c8802ffb [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 79d1b3f1a73945fe9d116354bf26157f (127.7.221.67:42171), 4a67bc06701742edb8d77170ed4da410 (127.7.221.66:45859)
03:47:53.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.248885 8606 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "f35afbde3ffe42e9aa2a447b50aed40c" candidate_uuid: "ef30ee403f0246569128f239c8802ffb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4a67bc06701742edb8d77170ed4da410" is_pre_election: true
03:47:53.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.248885 8752 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "f35afbde3ffe42e9aa2a447b50aed40c" candidate_uuid: "ef30ee403f0246569128f239c8802ffb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "79d1b3f1a73945fe9d116354bf26157f" is_pre_election: true
03:47:53.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.249540 8752 raft_consensus.cc:2466] T f35afbde3ffe42e9aa2a447b50aed40c P 79d1b3f1a73945fe9d116354bf26157f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ef30ee403f0246569128f239c8802ffb in term 0.
03:47:53.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.249539 8606 raft_consensus.cc:2466] T f35afbde3ffe42e9aa2a447b50aed40c P 4a67bc06701742edb8d77170ed4da410 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ef30ee403f0246569128f239c8802ffb in term 0.
03:47:53.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.250488 8382 leader_election.cc:304] T f35afbde3ffe42e9aa2a447b50aed40c P ef30ee403f0246569128f239c8802ffb [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 3 yes votes; 0 no votes. yes voters: 4a67bc06701742edb8d77170ed4da410, 79d1b3f1a73945fe9d116354bf26157f, ef30ee403f0246569128f239c8802ffb; no voters:
03:47:53.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.251006 8833 raft_consensus.cc:2802] T f35afbde3ffe42e9aa2a447b50aed40c P ef30ee403f0246569128f239c8802ffb [term 0 FOLLOWER]: Leader pre-election won for term 1
03:47:53.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.251168 8833 raft_consensus.cc:491] T f35afbde3ffe42e9aa2a447b50aed40c P ef30ee403f0246569128f239c8802ffb [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:47:53.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.251240 8833 raft_consensus.cc:3058] T f35afbde3ffe42e9aa2a447b50aed40c P ef30ee403f0246569128f239c8802ffb [term 0 FOLLOWER]: Advancing to term 1
03:47:53.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.253680 8833 raft_consensus.cc:513] T f35afbde3ffe42e9aa2a447b50aed40c P ef30ee403f0246569128f239c8802ffb [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef30ee403f0246569128f239c8802ffb" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34863 } } peers { permanent_uuid: "79d1b3f1a73945fe9d116354bf26157f" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42171 } } peers { permanent_uuid: "4a67bc06701742edb8d77170ed4da410" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 45859 } }
03:47:53.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.254496 8833 leader_election.cc:290] T f35afbde3ffe42e9aa2a447b50aed40c P ef30ee403f0246569128f239c8802ffb [CANDIDATE]: Term 1 election: Requested vote from peers 79d1b3f1a73945fe9d116354bf26157f (127.7.221.67:42171), 4a67bc06701742edb8d77170ed4da410 (127.7.221.66:45859)
03:47:53.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.255152 8606 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "f35afbde3ffe42e9aa2a447b50aed40c" candidate_uuid: "ef30ee403f0246569128f239c8802ffb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4a67bc06701742edb8d77170ed4da410"
03:47:53.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.255148 8752 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "f35afbde3ffe42e9aa2a447b50aed40c" candidate_uuid: "ef30ee403f0246569128f239c8802ffb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "79d1b3f1a73945fe9d116354bf26157f"
03:47:53.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.255358 8752 raft_consensus.cc:3058] T f35afbde3ffe42e9aa2a447b50aed40c P 79d1b3f1a73945fe9d116354bf26157f [term 0 FOLLOWER]: Advancing to term 1
03:47:53.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.255381 8606 raft_consensus.cc:3058] T f35afbde3ffe42e9aa2a447b50aed40c P 4a67bc06701742edb8d77170ed4da410 [term 0 FOLLOWER]: Advancing to term 1
03:47:53.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.257901 8606 raft_consensus.cc:2466] T f35afbde3ffe42e9aa2a447b50aed40c P 4a67bc06701742edb8d77170ed4da410 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ef30ee403f0246569128f239c8802ffb in term 1.
03:47:53.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.257925 8752 raft_consensus.cc:2466] T f35afbde3ffe42e9aa2a447b50aed40c P 79d1b3f1a73945fe9d116354bf26157f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ef30ee403f0246569128f239c8802ffb in term 1.
03:47:53.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.258548 8382 leader_election.cc:304] T f35afbde3ffe42e9aa2a447b50aed40c P ef30ee403f0246569128f239c8802ffb [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4a67bc06701742edb8d77170ed4da410, ef30ee403f0246569128f239c8802ffb; no voters:
03:47:53.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.258957 8833 raft_consensus.cc:2802] T f35afbde3ffe42e9aa2a447b50aed40c P ef30ee403f0246569128f239c8802ffb [term 1 FOLLOWER]: Leader election won for term 1
03:47:53.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.259464 8833 raft_consensus.cc:695] T f35afbde3ffe42e9aa2a447b50aed40c P ef30ee403f0246569128f239c8802ffb [term 1 LEADER]: Becoming Leader. State: Replica: ef30ee403f0246569128f239c8802ffb, State: Running, Role: LEADER
03:47:53.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.260107 8833 consensus_queue.cc:237] T f35afbde3ffe42e9aa2a447b50aed40c P ef30ee403f0246569128f239c8802ffb [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef30ee403f0246569128f239c8802ffb" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34863 } } peers { permanent_uuid: "79d1b3f1a73945fe9d116354bf26157f" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42171 } } peers { permanent_uuid: "4a67bc06701742edb8d77170ed4da410" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 45859 } }
03:47:53.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.266602 8259 catalog_manager.cc:5582] T f35afbde3ffe42e9aa2a447b50aed40c P ef30ee403f0246569128f239c8802ffb reported cstate change: term changed from 0 to 1, leader changed from <none> to ef30ee403f0246569128f239c8802ffb (127.7.221.65). New cstate: current_term: 1 leader_uuid: "ef30ee403f0246569128f239c8802ffb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef30ee403f0246569128f239c8802ffb" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34863 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "79d1b3f1a73945fe9d116354bf26157f" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42171 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4a67bc06701742edb8d77170ed4da410" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 45859 } health_report { overall_health: UNKNOWN } } }
03:47:53.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:53.327819 8657 tablet.cc:2378] T 729fc2d921ee42a0b407222da665b01c P 4a67bc06701742edb8d77170ed4da410: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:47:53.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.365211 8833 raft_consensus.cc:491] T 729fc2d921ee42a0b407222da665b01c P ef30ee403f0246569128f239c8802ffb [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:47:53.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.365445 8833 raft_consensus.cc:513] T 729fc2d921ee42a0b407222da665b01c P ef30ee403f0246569128f239c8802ffb [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef30ee403f0246569128f239c8802ffb" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34863 } } peers { permanent_uuid: "4a67bc06701742edb8d77170ed4da410" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 45859 } } peers { permanent_uuid: "79d1b3f1a73945fe9d116354bf26157f" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42171 } }
03:47:53.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.366115 8833 leader_election.cc:290] T 729fc2d921ee42a0b407222da665b01c P ef30ee403f0246569128f239c8802ffb [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4a67bc06701742edb8d77170ed4da410 (127.7.221.66:45859), 79d1b3f1a73945fe9d116354bf26157f (127.7.221.67:42171)
03:47:53.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.366827 8606 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "729fc2d921ee42a0b407222da665b01c" candidate_uuid: "ef30ee403f0246569128f239c8802ffb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4a67bc06701742edb8d77170ed4da410" is_pre_election: true
03:47:53.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.366829 8752 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "729fc2d921ee42a0b407222da665b01c" candidate_uuid: "ef30ee403f0246569128f239c8802ffb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "79d1b3f1a73945fe9d116354bf26157f" is_pre_election: true
03:47:53.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.367071 8606 raft_consensus.cc:2466] T 729fc2d921ee42a0b407222da665b01c P 4a67bc06701742edb8d77170ed4da410 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ef30ee403f0246569128f239c8802ffb in term 0.
03:47:53.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.367089 8752 raft_consensus.cc:2466] T 729fc2d921ee42a0b407222da665b01c P 79d1b3f1a73945fe9d116354bf26157f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ef30ee403f0246569128f239c8802ffb in term 0.
03:47:53.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.367585 8382 leader_election.cc:304] T 729fc2d921ee42a0b407222da665b01c P ef30ee403f0246569128f239c8802ffb [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 3 yes votes; 0 no votes. yes voters: 4a67bc06701742edb8d77170ed4da410, 79d1b3f1a73945fe9d116354bf26157f, ef30ee403f0246569128f239c8802ffb; no voters:
03:47:53.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.367972 8833 raft_consensus.cc:2802] T 729fc2d921ee42a0b407222da665b01c P ef30ee403f0246569128f239c8802ffb [term 0 FOLLOWER]: Leader pre-election won for term 1
03:47:53.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.368084 8833 raft_consensus.cc:491] T 729fc2d921ee42a0b407222da665b01c P ef30ee403f0246569128f239c8802ffb [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:47:53.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.368197 8833 raft_consensus.cc:3058] T 729fc2d921ee42a0b407222da665b01c P ef30ee403f0246569128f239c8802ffb [term 0 FOLLOWER]: Advancing to term 1
03:47:53.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.370467 8833 raft_consensus.cc:513] T 729fc2d921ee42a0b407222da665b01c P ef30ee403f0246569128f239c8802ffb [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef30ee403f0246569128f239c8802ffb" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34863 } } peers { permanent_uuid: "4a67bc06701742edb8d77170ed4da410" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 45859 } } peers { permanent_uuid: "79d1b3f1a73945fe9d116354bf26157f" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42171 } }
03:47:53.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.371289 8833 leader_election.cc:290] T 729fc2d921ee42a0b407222da665b01c P ef30ee403f0246569128f239c8802ffb [CANDIDATE]: Term 1 election: Requested vote from peers 4a67bc06701742edb8d77170ed4da410 (127.7.221.66:45859), 79d1b3f1a73945fe9d116354bf26157f (127.7.221.67:42171)
03:47:53.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.371835 8606 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "729fc2d921ee42a0b407222da665b01c" candidate_uuid: "ef30ee403f0246569128f239c8802ffb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4a67bc06701742edb8d77170ed4da410"
03:47:53.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.371863 8752 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "729fc2d921ee42a0b407222da665b01c" candidate_uuid: "ef30ee403f0246569128f239c8802ffb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "79d1b3f1a73945fe9d116354bf26157f"
03:47:53.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.372063 8606 raft_consensus.cc:3058] T 729fc2d921ee42a0b407222da665b01c P 4a67bc06701742edb8d77170ed4da410 [term 0 FOLLOWER]: Advancing to term 1
03:47:53.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.372087 8752 raft_consensus.cc:3058] T 729fc2d921ee42a0b407222da665b01c P 79d1b3f1a73945fe9d116354bf26157f [term 0 FOLLOWER]: Advancing to term 1
03:47:53.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.374143 8752 raft_consensus.cc:2466] T 729fc2d921ee42a0b407222da665b01c P 79d1b3f1a73945fe9d116354bf26157f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ef30ee403f0246569128f239c8802ffb in term 1.
03:47:53.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.374173 8606 raft_consensus.cc:2466] T 729fc2d921ee42a0b407222da665b01c P 4a67bc06701742edb8d77170ed4da410 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ef30ee403f0246569128f239c8802ffb in term 1.
03:47:53.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.374831 8381 leader_election.cc:304] T 729fc2d921ee42a0b407222da665b01c P ef30ee403f0246569128f239c8802ffb [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 3 yes votes; 0 no votes. yes voters: 4a67bc06701742edb8d77170ed4da410, 79d1b3f1a73945fe9d116354bf26157f, ef30ee403f0246569128f239c8802ffb; no voters:
03:47:53.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.375240 8833 raft_consensus.cc:2802] T 729fc2d921ee42a0b407222da665b01c P ef30ee403f0246569128f239c8802ffb [term 1 FOLLOWER]: Leader election won for term 1
03:47:53.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.375440 8833 raft_consensus.cc:695] T 729fc2d921ee42a0b407222da665b01c P ef30ee403f0246569128f239c8802ffb [term 1 LEADER]: Becoming Leader. State: Replica: ef30ee403f0246569128f239c8802ffb, State: Running, Role: LEADER
03:47:53.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.375798 8833 consensus_queue.cc:237] T 729fc2d921ee42a0b407222da665b01c P ef30ee403f0246569128f239c8802ffb [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef30ee403f0246569128f239c8802ffb" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34863 } } peers { permanent_uuid: "4a67bc06701742edb8d77170ed4da410" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 45859 } } peers { permanent_uuid: "79d1b3f1a73945fe9d116354bf26157f" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42171 } }
03:47:53.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.379097 8259 catalog_manager.cc:5582] T 729fc2d921ee42a0b407222da665b01c P ef30ee403f0246569128f239c8802ffb reported cstate change: term changed from 0 to 1, leader changed from <none> to ef30ee403f0246569128f239c8802ffb (127.7.221.65). New cstate: current_term: 1 leader_uuid: "ef30ee403f0246569128f239c8802ffb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef30ee403f0246569128f239c8802ffb" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34863 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4a67bc06701742edb8d77170ed4da410" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 45859 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "79d1b3f1a73945fe9d116354bf26157f" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42171 } health_report { overall_health: UNKNOWN } } }
03:47:53.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.601522 8259 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:54708:
03:47:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
03:47:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
03:47:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:47:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
03:47:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
03:47:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
03:47:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:47:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
03:47:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
03:47:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:47:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:47:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:47:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:47:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
03:47:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
03:47:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
03:47:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:47:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
03:47:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
03:47:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:47:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:47:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:47:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:47:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
03:47:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
03:47:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
03:47:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:47:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
03:47:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
03:47:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:47:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:47:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:47:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:47:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
03:47:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
03:47:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
03:47:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:47:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
03:47:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
03:47:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:47:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:47:53.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:47:53.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:47:53.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
03:47:53.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
03:47:53.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
03:47:53.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
03:47:53.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:47:53.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
03:47:53.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
03:47:53.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:47:53.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
03:47:53.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:47:53.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:47:53.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:47:53.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
03:47:53.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:53.602939 8259 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
03:47:53.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.618782 8727 tablet_service.cc:1468] Processing CreateTablet for tablet 84d3092f319b4ee2808165b03e84a2fe (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=b19531f9f14141c7b6e8abf776a43683]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
03:47:53.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.619726 8727 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 84d3092f319b4ee2808165b03e84a2fe. 1 dirs total, 0 dirs full, 0 dirs failed
03:47:53.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.619386 8432 tablet_service.cc:1468] Processing CreateTablet for tablet 84d3092f319b4ee2808165b03e84a2fe (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=b19531f9f14141c7b6e8abf776a43683]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
03:47:53.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.620126 8432 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 84d3092f319b4ee2808165b03e84a2fe. 1 dirs total, 0 dirs full, 0 dirs failed
03:47:53.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.621435 8581 tablet_service.cc:1468] Processing CreateTablet for tablet 84d3092f319b4ee2808165b03e84a2fe (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=b19531f9f14141c7b6e8abf776a43683]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
03:47:53.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.622429 8581 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 84d3092f319b4ee2808165b03e84a2fe. 1 dirs total, 0 dirs full, 0 dirs failed
03:47:53.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.625983 8825 tablet_bootstrap.cc:492] T 84d3092f319b4ee2808165b03e84a2fe P ef30ee403f0246569128f239c8802ffb: Bootstrap starting.
03:47:53.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.628065 8825 tablet_bootstrap.cc:654] T 84d3092f319b4ee2808165b03e84a2fe P ef30ee403f0246569128f239c8802ffb: Neither blocks nor log segments found. Creating new log.
03:47:53.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.634822 8853 tablet_bootstrap.cc:492] T 84d3092f319b4ee2808165b03e84a2fe P 79d1b3f1a73945fe9d116354bf26157f: Bootstrap starting.
03:47:53.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.637300 8825 tablet_bootstrap.cc:492] T 84d3092f319b4ee2808165b03e84a2fe P ef30ee403f0246569128f239c8802ffb: No bootstrap required, opened a new log
03:47:53.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.637529 8825 ts_tablet_manager.cc:1397] T 84d3092f319b4ee2808165b03e84a2fe P ef30ee403f0246569128f239c8802ffb: Time spent bootstrapping tablet: real 0.012s user 0.009s sys 0.001s
03:47:53.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.638604 8825 raft_consensus.cc:357] T 84d3092f319b4ee2808165b03e84a2fe P ef30ee403f0246569128f239c8802ffb [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef30ee403f0246569128f239c8802ffb" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34863 } } peers { permanent_uuid: "4a67bc06701742edb8d77170ed4da410" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 45859 } } peers { permanent_uuid: "79d1b3f1a73945fe9d116354bf26157f" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42171 } }
03:47:53.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.638832 8825 raft_consensus.cc:383] T 84d3092f319b4ee2808165b03e84a2fe P ef30ee403f0246569128f239c8802ffb [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:47:53.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.638926 8825 raft_consensus.cc:738] T 84d3092f319b4ee2808165b03e84a2fe P ef30ee403f0246569128f239c8802ffb [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ef30ee403f0246569128f239c8802ffb, State: Initialized, Role: FOLLOWER
03:47:53.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.639189 8825 consensus_queue.cc:260] T 84d3092f319b4ee2808165b03e84a2fe P ef30ee403f0246569128f239c8802ffb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef30ee403f0246569128f239c8802ffb" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34863 } } peers { permanent_uuid: "4a67bc06701742edb8d77170ed4da410" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 45859 } } peers { permanent_uuid: "79d1b3f1a73945fe9d116354bf26157f" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42171 } }
03:47:53.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.639325 8854 tablet_bootstrap.cc:492] T 84d3092f319b4ee2808165b03e84a2fe P 4a67bc06701742edb8d77170ed4da410: Bootstrap starting.
03:47:53.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.639782 8825 ts_tablet_manager.cc:1428] T 84d3092f319b4ee2808165b03e84a2fe P ef30ee403f0246569128f239c8802ffb: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
03:47:53.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.639729 8833 tablet_replica.cc:440] T 84d3092f319b4ee2808165b03e84a2fe P ef30ee403f0246569128f239c8802ffb: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef30ee403f0246569128f239c8802ffb" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34863 } } peers { permanent_uuid: "4a67bc06701742edb8d77170ed4da410" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 45859 } } peers { permanent_uuid: "79d1b3f1a73945fe9d116354bf26157f" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42171 } } }
03:47:53.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.639966 8833 tablet_replica.cc:443] T 84d3092f319b4ee2808165b03e84a2fe P ef30ee403f0246569128f239c8802ffb: This TxnStatusTablet replica's current role is: FOLLOWER
03:47:53.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.641119 8854 tablet_bootstrap.cc:654] T 84d3092f319b4ee2808165b03e84a2fe P 4a67bc06701742edb8d77170ed4da410: Neither blocks nor log segments found. Creating new log.
03:47:53.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.642514 8853 tablet_bootstrap.cc:654] T 84d3092f319b4ee2808165b03e84a2fe P 79d1b3f1a73945fe9d116354bf26157f: Neither blocks nor log segments found. Creating new log.
03:47:53.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.645227 8854 tablet_bootstrap.cc:492] T 84d3092f319b4ee2808165b03e84a2fe P 4a67bc06701742edb8d77170ed4da410: No bootstrap required, opened a new log
03:47:53.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.645442 8854 ts_tablet_manager.cc:1397] T 84d3092f319b4ee2808165b03e84a2fe P 4a67bc06701742edb8d77170ed4da410: Time spent bootstrapping tablet: real 0.006s user 0.003s sys 0.000s
03:47:53.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.645457 8853 tablet_bootstrap.cc:492] T 84d3092f319b4ee2808165b03e84a2fe P 79d1b3f1a73945fe9d116354bf26157f: No bootstrap required, opened a new log
03:47:53.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.645628 8853 ts_tablet_manager.cc:1397] T 84d3092f319b4ee2808165b03e84a2fe P 79d1b3f1a73945fe9d116354bf26157f: Time spent bootstrapping tablet: real 0.011s user 0.004s sys 0.000s
03:47:53.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.646317 8853 raft_consensus.cc:357] T 84d3092f319b4ee2808165b03e84a2fe P 79d1b3f1a73945fe9d116354bf26157f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef30ee403f0246569128f239c8802ffb" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34863 } } peers { permanent_uuid: "4a67bc06701742edb8d77170ed4da410" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 45859 } } peers { permanent_uuid: "79d1b3f1a73945fe9d116354bf26157f" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42171 } }
03:47:53.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.646289 8854 raft_consensus.cc:357] T 84d3092f319b4ee2808165b03e84a2fe P 4a67bc06701742edb8d77170ed4da410 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef30ee403f0246569128f239c8802ffb" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34863 } } peers { permanent_uuid: "4a67bc06701742edb8d77170ed4da410" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 45859 } } peers { permanent_uuid: "79d1b3f1a73945fe9d116354bf26157f" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42171 } }
03:47:53.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.646479 8853 raft_consensus.cc:383] T 84d3092f319b4ee2808165b03e84a2fe P 79d1b3f1a73945fe9d116354bf26157f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:47:53.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.646485 8854 raft_consensus.cc:383] T 84d3092f319b4ee2808165b03e84a2fe P 4a67bc06701742edb8d77170ed4da410 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:47:53.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.646586 8854 raft_consensus.cc:738] T 84d3092f319b4ee2808165b03e84a2fe P 4a67bc06701742edb8d77170ed4da410 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4a67bc06701742edb8d77170ed4da410, State: Initialized, Role: FOLLOWER
03:47:53.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.646581 8853 raft_consensus.cc:738] T 84d3092f319b4ee2808165b03e84a2fe P 79d1b3f1a73945fe9d116354bf26157f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 79d1b3f1a73945fe9d116354bf26157f, State: Initialized, Role: FOLLOWER
03:47:53.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.646865 8854 consensus_queue.cc:260] T 84d3092f319b4ee2808165b03e84a2fe P 4a67bc06701742edb8d77170ed4da410 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef30ee403f0246569128f239c8802ffb" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34863 } } peers { permanent_uuid: "4a67bc06701742edb8d77170ed4da410" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 45859 } } peers { permanent_uuid: "79d1b3f1a73945fe9d116354bf26157f" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42171 } }
03:47:53.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.646896 8853 consensus_queue.cc:260] T 84d3092f319b4ee2808165b03e84a2fe P 79d1b3f1a73945fe9d116354bf26157f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef30ee403f0246569128f239c8802ffb" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34863 } } peers { permanent_uuid: "4a67bc06701742edb8d77170ed4da410" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 45859 } } peers { permanent_uuid: "79d1b3f1a73945fe9d116354bf26157f" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42171 } }
03:47:53.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.649546 8833 raft_consensus.cc:491] T 84d3092f319b4ee2808165b03e84a2fe P ef30ee403f0246569128f239c8802ffb [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:47:53.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.649770 8833 raft_consensus.cc:513] T 84d3092f319b4ee2808165b03e84a2fe P ef30ee403f0246569128f239c8802ffb [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef30ee403f0246569128f239c8802ffb" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34863 } } peers { permanent_uuid: "4a67bc06701742edb8d77170ed4da410" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 45859 } } peers { permanent_uuid: "79d1b3f1a73945fe9d116354bf26157f" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42171 } }
03:47:53.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.650406 8854 ts_tablet_manager.cc:1428] T 84d3092f319b4ee2808165b03e84a2fe P 4a67bc06701742edb8d77170ed4da410: Time spent starting tablet: real 0.005s user 0.002s sys 0.000s
03:47:53.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.650573 8833 leader_election.cc:290] T 84d3092f319b4ee2808165b03e84a2fe P ef30ee403f0246569128f239c8802ffb [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4a67bc06701742edb8d77170ed4da410 (127.7.221.66:45859), 79d1b3f1a73945fe9d116354bf26157f (127.7.221.67:42171)
03:47:53.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.652122 8606 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "84d3092f319b4ee2808165b03e84a2fe" candidate_uuid: "ef30ee403f0246569128f239c8802ffb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4a67bc06701742edb8d77170ed4da410" is_pre_election: true
03:47:53.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.652135 8752 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "84d3092f319b4ee2808165b03e84a2fe" candidate_uuid: "ef30ee403f0246569128f239c8802ffb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "79d1b3f1a73945fe9d116354bf26157f" is_pre_election: true
03:47:53.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.652346 8606 raft_consensus.cc:2466] T 84d3092f319b4ee2808165b03e84a2fe P 4a67bc06701742edb8d77170ed4da410 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ef30ee403f0246569128f239c8802ffb in term 0.
03:47:53.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.652354 8752 raft_consensus.cc:2466] T 84d3092f319b4ee2808165b03e84a2fe P 79d1b3f1a73945fe9d116354bf26157f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ef30ee403f0246569128f239c8802ffb in term 0.
03:47:53.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.653033 8382 leader_election.cc:304] T 84d3092f319b4ee2808165b03e84a2fe P ef30ee403f0246569128f239c8802ffb [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4a67bc06701742edb8d77170ed4da410, ef30ee403f0246569128f239c8802ffb; no voters:
03:47:53.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.653381 8833 raft_consensus.cc:2802] T 84d3092f319b4ee2808165b03e84a2fe P ef30ee403f0246569128f239c8802ffb [term 0 FOLLOWER]: Leader pre-election won for term 1
03:47:53.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.653524 8833 raft_consensus.cc:491] T 84d3092f319b4ee2808165b03e84a2fe P ef30ee403f0246569128f239c8802ffb [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:47:53.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.653604 8833 raft_consensus.cc:3058] T 84d3092f319b4ee2808165b03e84a2fe P ef30ee403f0246569128f239c8802ffb [term 0 FOLLOWER]: Advancing to term 1
03:47:53.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.657327 8833 raft_consensus.cc:513] T 84d3092f319b4ee2808165b03e84a2fe P ef30ee403f0246569128f239c8802ffb [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef30ee403f0246569128f239c8802ffb" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34863 } } peers { permanent_uuid: "4a67bc06701742edb8d77170ed4da410" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 45859 } } peers { permanent_uuid: "79d1b3f1a73945fe9d116354bf26157f" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42171 } }
03:47:53.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.657629 8857 tablet_replica.cc:440] T 84d3092f319b4ee2808165b03e84a2fe P 4a67bc06701742edb8d77170ed4da410: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef30ee403f0246569128f239c8802ffb" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34863 } } peers { permanent_uuid: "4a67bc06701742edb8d77170ed4da410" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 45859 } } peers { permanent_uuid: "79d1b3f1a73945fe9d116354bf26157f" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42171 } } }
03:47:53.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.658396 8857 tablet_replica.cc:443] T 84d3092f319b4ee2808165b03e84a2fe P 4a67bc06701742edb8d77170ed4da410: This TxnStatusTablet replica's current role is: FOLLOWER
03:47:53.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.658748 8606 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "84d3092f319b4ee2808165b03e84a2fe" candidate_uuid: "ef30ee403f0246569128f239c8802ffb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4a67bc06701742edb8d77170ed4da410"
03:47:53.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.658957 8606 raft_consensus.cc:3058] T 84d3092f319b4ee2808165b03e84a2fe P 4a67bc06701742edb8d77170ed4da410 [term 0 FOLLOWER]: Advancing to term 1
03:47:53.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.659333 8853 ts_tablet_manager.cc:1428] T 84d3092f319b4ee2808165b03e84a2fe P 79d1b3f1a73945fe9d116354bf26157f: Time spent starting tablet: real 0.014s user 0.002s sys 0.006s
03:47:53.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.662159 8833 leader_election.cc:290] T 84d3092f319b4ee2808165b03e84a2fe P ef30ee403f0246569128f239c8802ffb [CANDIDATE]: Term 1 election: Requested vote from peers 4a67bc06701742edb8d77170ed4da410 (127.7.221.66:45859), 79d1b3f1a73945fe9d116354bf26157f (127.7.221.67:42171)
03:47:53.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.662163 8606 raft_consensus.cc:2466] T 84d3092f319b4ee2808165b03e84a2fe P 4a67bc06701742edb8d77170ed4da410 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ef30ee403f0246569128f239c8802ffb in term 1.
03:47:53.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.662461 8752 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "84d3092f319b4ee2808165b03e84a2fe" candidate_uuid: "ef30ee403f0246569128f239c8802ffb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "79d1b3f1a73945fe9d116354bf26157f"
03:47:53.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.662703 8752 raft_consensus.cc:3058] T 84d3092f319b4ee2808165b03e84a2fe P 79d1b3f1a73945fe9d116354bf26157f [term 0 FOLLOWER]: Advancing to term 1
03:47:53.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.662716 8382 leader_election.cc:304] T 84d3092f319b4ee2808165b03e84a2fe P ef30ee403f0246569128f239c8802ffb [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4a67bc06701742edb8d77170ed4da410, ef30ee403f0246569128f239c8802ffb; no voters:
03:47:53.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.663110 8833 raft_consensus.cc:2802] T 84d3092f319b4ee2808165b03e84a2fe P ef30ee403f0246569128f239c8802ffb [term 1 FOLLOWER]: Leader election won for term 1
03:47:53.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.664023 8833 raft_consensus.cc:695] T 84d3092f319b4ee2808165b03e84a2fe P ef30ee403f0246569128f239c8802ffb [term 1 LEADER]: Becoming Leader. State: Replica: ef30ee403f0246569128f239c8802ffb, State: Running, Role: LEADER
03:47:53.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.664341 8833 consensus_queue.cc:237] T 84d3092f319b4ee2808165b03e84a2fe P ef30ee403f0246569128f239c8802ffb [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef30ee403f0246569128f239c8802ffb" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34863 } } peers { permanent_uuid: "4a67bc06701742edb8d77170ed4da410" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 45859 } } peers { permanent_uuid: "79d1b3f1a73945fe9d116354bf26157f" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42171 } }
03:47:53.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.666534 8752 raft_consensus.cc:2466] T 84d3092f319b4ee2808165b03e84a2fe P 79d1b3f1a73945fe9d116354bf26157f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ef30ee403f0246569128f239c8802ffb in term 1.
03:47:53.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.668385 8858 tablet_replica.cc:440] T 84d3092f319b4ee2808165b03e84a2fe P 79d1b3f1a73945fe9d116354bf26157f: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef30ee403f0246569128f239c8802ffb" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34863 } } peers { permanent_uuid: "4a67bc06701742edb8d77170ed4da410" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 45859 } } peers { permanent_uuid: "79d1b3f1a73945fe9d116354bf26157f" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42171 } } }
03:47:53.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.668781 8858 tablet_replica.cc:443] T 84d3092f319b4ee2808165b03e84a2fe P 79d1b3f1a73945fe9d116354bf26157f: This TxnStatusTablet replica's current role is: FOLLOWER
03:47:53.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.669728 8838 tablet_replica.cc:440] T 84d3092f319b4ee2808165b03e84a2fe P ef30ee403f0246569128f239c8802ffb: TxnStatusTablet state changed. Reason: New leader ef30ee403f0246569128f239c8802ffb. Latest consensus state: current_term: 1 leader_uuid: "ef30ee403f0246569128f239c8802ffb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef30ee403f0246569128f239c8802ffb" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34863 } } peers { permanent_uuid: "4a67bc06701742edb8d77170ed4da410" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 45859 } } peers { permanent_uuid: "79d1b3f1a73945fe9d116354bf26157f" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42171 } } }
03:47:53.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.670002 8838 tablet_replica.cc:443] T 84d3092f319b4ee2808165b03e84a2fe P ef30ee403f0246569128f239c8802ffb: This TxnStatusTablet replica's current role is: LEADER
03:47:53.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.670868 8860 txn_status_manager.cc:874] Waiting until node catch up with all replicated operations in previous term...
03:47:53.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.672688 8860 txn_status_manager.cc:930] Loading transaction status metadata into memory...
03:47:53.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.672444 8258 catalog_manager.cc:5582] T 84d3092f319b4ee2808165b03e84a2fe P ef30ee403f0246569128f239c8802ffb reported cstate change: term changed from 0 to 1, leader changed from <none> to ef30ee403f0246569128f239c8802ffb (127.7.221.65). New cstate: current_term: 1 leader_uuid: "ef30ee403f0246569128f239c8802ffb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef30ee403f0246569128f239c8802ffb" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34863 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4a67bc06701742edb8d77170ed4da410" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 45859 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "79d1b3f1a73945fe9d116354bf26157f" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42171 } health_report { overall_health: UNKNOWN } } }
03:47:53.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.708603 8752 raft_consensus.cc:1273] T 84d3092f319b4ee2808165b03e84a2fe P 79d1b3f1a73945fe9d116354bf26157f [term 1 FOLLOWER]: Refusing update from remote peer ef30ee403f0246569128f239c8802ffb: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:47:53.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.708592 8606 raft_consensus.cc:1273] T 84d3092f319b4ee2808165b03e84a2fe P 4a67bc06701742edb8d77170ed4da410 [term 1 FOLLOWER]: Refusing update from remote peer ef30ee403f0246569128f239c8802ffb: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:47:53.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.709869 8838 consensus_queue.cc:1035] T 84d3092f319b4ee2808165b03e84a2fe P ef30ee403f0246569128f239c8802ffb [LEADER]: Connected to new peer: Peer: permanent_uuid: "4a67bc06701742edb8d77170ed4da410" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 45859 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:47:53.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.710361 8833 consensus_queue.cc:1035] T 84d3092f319b4ee2808165b03e84a2fe P ef30ee403f0246569128f239c8802ffb [LEADER]: Connected to new peer: Peer: permanent_uuid: "79d1b3f1a73945fe9d116354bf26157f" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42171 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:47:53.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.726408 8858 tablet_replica.cc:440] T 84d3092f319b4ee2808165b03e84a2fe P 79d1b3f1a73945fe9d116354bf26157f: TxnStatusTablet state changed. Reason: New leader ef30ee403f0246569128f239c8802ffb. Latest consensus state: current_term: 1 leader_uuid: "ef30ee403f0246569128f239c8802ffb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef30ee403f0246569128f239c8802ffb" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34863 } } peers { permanent_uuid: "4a67bc06701742edb8d77170ed4da410" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 45859 } } peers { permanent_uuid: "79d1b3f1a73945fe9d116354bf26157f" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42171 } } }
03:47:53.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.726662 8858 tablet_replica.cc:443] T 84d3092f319b4ee2808165b03e84a2fe P 79d1b3f1a73945fe9d116354bf26157f: This TxnStatusTablet replica's current role is: FOLLOWER
03:47:53.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.729091 8857 tablet_replica.cc:440] T 84d3092f319b4ee2808165b03e84a2fe P 4a67bc06701742edb8d77170ed4da410: TxnStatusTablet state changed. Reason: New leader ef30ee403f0246569128f239c8802ffb. Latest consensus state: current_term: 1 leader_uuid: "ef30ee403f0246569128f239c8802ffb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef30ee403f0246569128f239c8802ffb" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34863 } } peers { permanent_uuid: "4a67bc06701742edb8d77170ed4da410" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 45859 } } peers { permanent_uuid: "79d1b3f1a73945fe9d116354bf26157f" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42171 } } }
03:47:53.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.729362 8857 tablet_replica.cc:443] T 84d3092f319b4ee2808165b03e84a2fe P 4a67bc06701742edb8d77170ed4da410: This TxnStatusTablet replica's current role is: FOLLOWER
03:47:53.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.732667 8864 mvcc.cc:204] Tried to move back new op lower bound from 7171372129098022912 to 7171372128931876864. Current Snapshot: MvccSnapshot[applied={T|T < 7171372129098022912}]
03:47:53.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.733897 8838 tablet_replica.cc:440] T 84d3092f319b4ee2808165b03e84a2fe P ef30ee403f0246569128f239c8802ffb: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "ef30ee403f0246569128f239c8802ffb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef30ee403f0246569128f239c8802ffb" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34863 } } peers { permanent_uuid: "4a67bc06701742edb8d77170ed4da410" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 45859 } } peers { permanent_uuid: "79d1b3f1a73945fe9d116354bf26157f" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42171 } } }
03:47:53.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.734179 8838 tablet_replica.cc:443] T 84d3092f319b4ee2808165b03e84a2fe P ef30ee403f0246569128f239c8802ffb: This TxnStatusTablet replica's current role is: LEADER
03:47:53.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.734474 8838 tablet_replica.cc:440] T 84d3092f319b4ee2808165b03e84a2fe P ef30ee403f0246569128f239c8802ffb: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "ef30ee403f0246569128f239c8802ffb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef30ee403f0246569128f239c8802ffb" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34863 } } peers { permanent_uuid: "4a67bc06701742edb8d77170ed4da410" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 45859 } } peers { permanent_uuid: "79d1b3f1a73945fe9d116354bf26157f" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42171 } } }
03:47:53.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.734681 8838 tablet_replica.cc:443] T 84d3092f319b4ee2808165b03e84a2fe P ef30ee403f0246569128f239c8802ffb: This TxnStatusTablet replica's current role is: LEADER
03:47:53.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.739718 8857 tablet_replica.cc:440] T 84d3092f319b4ee2808165b03e84a2fe P 4a67bc06701742edb8d77170ed4da410: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "ef30ee403f0246569128f239c8802ffb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef30ee403f0246569128f239c8802ffb" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34863 } } peers { permanent_uuid: "4a67bc06701742edb8d77170ed4da410" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 45859 } } peers { permanent_uuid: "79d1b3f1a73945fe9d116354bf26157f" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42171 } } }
03:47:53.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.739923 8857 tablet_replica.cc:443] T 84d3092f319b4ee2808165b03e84a2fe P 4a67bc06701742edb8d77170ed4da410: This TxnStatusTablet replica's current role is: FOLLOWER
03:47:53.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.740024 8866 mvcc.cc:204] Tried to move back new op lower bound from 7171372129098022912 to 7171372128931876864. Current Snapshot: MvccSnapshot[applied={T|T < 7171372129098022912}]
03:47:53.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.744148 8858 tablet_replica.cc:440] T 84d3092f319b4ee2808165b03e84a2fe P 79d1b3f1a73945fe9d116354bf26157f: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "ef30ee403f0246569128f239c8802ffb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef30ee403f0246569128f239c8802ffb" member_type: VOTER last_known_addr { host: "127.7.221.65" port: 34863 } } peers { permanent_uuid: "4a67bc06701742edb8d77170ed4da410" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 45859 } } peers { permanent_uuid: "79d1b3f1a73945fe9d116354bf26157f" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42171 } } }
03:47:53.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.744395 8858 tablet_replica.cc:443] T 84d3092f319b4ee2808165b03e84a2fe P 79d1b3f1a73945fe9d116354bf26157f: This TxnStatusTablet replica's current role is: FOLLOWER
03:47:53.746 [DEBUG - Time-limited test] (KuduTransaction.java:702) starting keepalive heartbeating with period 500 ms (txn ID 0)
03:47:53.749 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 1f8120a6dc0743938fb8db3498d07cae
03:47:53.751 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet f35afbde3ffe42e9aa2a447b50aed40c for table 'txn_manager_fallback' with partition [0x00000000, 0x00000001)
03:47:53.752 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 729fc2d921ee42a0b407222da665b01c for table 'txn_manager_fallback' with partition [0x00000001, 0x00000002)
03:47:53.752 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x00000000, upperBoundPartitionKey=0x00000001, ttl=299999, tablet=f35afbde3ffe42e9aa2a447b50aed40c@[4a67bc06701742edb8d77170ed4da410(127.7.221.66:45859),79d1b3f1a73945fe9d116354bf26157f(127.7.221.67:42171),ef30ee403f0246569128f239c8802ffb(127.7.221.65:34863)[L]]}, Tablet{lowerBoundPartitionKey=0x00000001, upperBoundPartitionKey=0x00000002, ttl=299999, tablet=729fc2d921ee42a0b407222da665b01c@[4a67bc06701742edb8d77170ed4da410(127.7.221.66:45859),79d1b3f1a73945fe9d116354bf26157f(127.7.221.67:42171),ef30ee403f0246569128f239c8802ffb(127.7.221.65:34863)[L]]}, NonCoveredRange{lowerBoundPartitionKey=0x00000002, upperBoundPartitionKey=0x, ttl=299999}]
03:47:53.752 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=2), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.7.221.124:35655, [2ms] Sub RPC GetTableLocations: received response from server master-127.7.221.124:35655: OK, deferred=null) after lookup
03:47:53.753 [DEBUG - kudu-nio-0] (Connection.java:748) [peer ef30ee403f0246569128f239c8802ffb(127.7.221.65:34863)] connecting to peer
03:47:53.753 [DEBUG - kudu-nio-1] (Connection.java:757) [peer ef30ee403f0246569128f239c8802ffb(127.7.221.65:34863)] Successfully connected to peer
03:47:53.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.760948 8833 consensus_queue.cc:1035] T f35afbde3ffe42e9aa2a447b50aed40c P ef30ee403f0246569128f239c8802ffb [LEADER]: Connected to new peer: Peer: permanent_uuid: "79d1b3f1a73945fe9d116354bf26157f" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42171 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:47:53.768 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x32851b33, L:/127.0.0.1:52878 - R:/127.7.221.65:34863] using TOKEN/null
03:47:53.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.779922 8838 consensus_queue.cc:1035] T f35afbde3ffe42e9aa2a447b50aed40c P ef30ee403f0246569128f239c8802ffb [LEADER]: Connected to new peer: Peer: permanent_uuid: "4a67bc06701742edb8d77170ed4da410" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 45859 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:47:53.841 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.7.221.124:35655
03:47:53.842 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
03:47:53.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.843878 8053 external_mini_cluster.cc:1661] Attempting to check leaks for /tmp/dist-test-taskqbWxBn/build/asan/bin/kudu pid 8225
03:47:53.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.845568 8299 generic_service.cc:196] Checking for leaks (request via RPC)
03:47:53.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.871829 8833 consensus_queue.cc:1035] T 729fc2d921ee42a0b407222da665b01c P ef30ee403f0246569128f239c8802ffb [LEADER]: Connected to new peer: Peer: permanent_uuid: "79d1b3f1a73945fe9d116354bf26157f" member_type: VOTER last_known_addr { host: "127.7.221.67" port: 42171 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:47:53.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:53.896252 8878 consensus_queue.cc:1035] T 729fc2d921ee42a0b407222da665b01c P ef30ee403f0246569128f239c8802ffb [LEADER]: Connected to new peer: Peer: permanent_uuid: "4a67bc06701742edb8d77170ed4da410" member_type: VOTER last_known_addr { host: "127.7.221.66" port: 45859 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:47:54.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.029664 8053 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskqbWxBn/build/asan/bin/kudu with pid 8225
03:47:54.060 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.7.221.124:35655(127.7.221.124:35655)] handling channelInactive
03:47:54.060 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.221.124:35655(127.7.221.124:35655)] cleaning up while in state READY due to: connection closed
03:47:54.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:54.061331 8653 heartbeater.cc:646] Failed to heartbeat to 127.7.221.124:35655 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv got EOF from 127.7.221.124:35655 (error 108)
03:47:54.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:54.061331 8798 heartbeater.cc:646] Failed to heartbeat to 127.7.221.124:35655 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv got EOF from 127.7.221.124:35655 (error 108)
03:47:54.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:54.062207 8506 heartbeater.cc:646] Failed to heartbeat to 127.7.221.124:35655 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv got EOF from 127.7.221.124:35655 (error 108)
03:47:54.063 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:47:54.254 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.7.221.124:35655(127.7.221.124:35655)] connecting to peer
03:47:54.256 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.256 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.221.124:35655(127.7.221.124:35655)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.256 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location master-127.7.221.124:35655(127.7.221.124:35655) for tablet Kudu Master: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.269 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.7.221.124:35655(127.7.221.124:35655)] connecting to peer
03:47:54.272 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.272 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.221.124:35655(127.7.221.124:35655)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.272 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.221.124:35655: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.273 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655; will retry
03:47:54.292 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.7.221.124:35655(127.7.221.124:35655)] connecting to peer
03:47:54.293 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.293 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.221.124:35655(127.7.221.124:35655)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.294 [INFO - pool-38-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.221.124:35655: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.294 [WARN - pool-38-thread-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655; will retry
03:47:54.314 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.7.221.124:35655(127.7.221.124:35655)] connecting to peer
03:47:54.316 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.316 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.221.124:35655(127.7.221.124:35655)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.316 [INFO - pool-38-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.221.124:35655: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.316 [WARN - pool-38-thread-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655; will retry
03:47:54.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.329527 8884 raft_consensus.cc:491] T 00000000000000000000000000000000 P 4c24d4441f6c47728564eedff9749c8f [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 59d0ebb2088d4fafbfc147165d7544f8)
03:47:54.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.339855 8884 raft_consensus.cc:513] T 00000000000000000000000000000000 P 4c24d4441f6c47728564eedff9749c8f [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c24d4441f6c47728564eedff9749c8f" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 43693 } } peers { permanent_uuid: "abe570d88d9e41929e679edcc29956c5" member_type: VOTER last_known_addr { host: "127.7.221.125" port: 33545 } } peers { permanent_uuid: "59d0ebb2088d4fafbfc147165d7544f8" member_type: VOTER last_known_addr { host: "127.7.221.124" port: 35655 } }
03:47:54.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.340049 8885 raft_consensus.cc:491] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 59d0ebb2088d4fafbfc147165d7544f8)
03:47:54.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.340270 8885 raft_consensus.cc:513] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c24d4441f6c47728564eedff9749c8f" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 43693 } } peers { permanent_uuid: "abe570d88d9e41929e679edcc29956c5" member_type: VOTER last_known_addr { host: "127.7.221.125" port: 33545 } } peers { permanent_uuid: "59d0ebb2088d4fafbfc147165d7544f8" member_type: VOTER last_known_addr { host: "127.7.221.124" port: 35655 } }
03:47:54.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.341853 8106 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "abe570d88d9e41929e679edcc29956c5" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "4c24d4441f6c47728564eedff9749c8f" is_pre_election: true
03:47:54.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.342176 8106 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 4c24d4441f6c47728564eedff9749c8f [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate abe570d88d9e41929e679edcc29956c5 in term 1.
03:47:54.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.342312 8186 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4c24d4441f6c47728564eedff9749c8f" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "abe570d88d9e41929e679edcc29956c5" is_pre_election: true
03:47:54.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.342621 8186 raft_consensus.cc:2466] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4c24d4441f6c47728564eedff9749c8f in term 1.
03:47:54.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.343062 8164 leader_election.cc:304] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4c24d4441f6c47728564eedff9749c8f, abe570d88d9e41929e679edcc29956c5; no voters:
03:47:54.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.343145 8083 leader_election.cc:304] T 00000000000000000000000000000000 P 4c24d4441f6c47728564eedff9749c8f [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4c24d4441f6c47728564eedff9749c8f, abe570d88d9e41929e679edcc29956c5; no voters:
03:47:54.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.347798 8884 leader_election.cc:290] T 00000000000000000000000000000000 P 4c24d4441f6c47728564eedff9749c8f [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers abe570d88d9e41929e679edcc29956c5 (127.7.221.125:33545), 59d0ebb2088d4fafbfc147165d7544f8 (127.7.221.124:35655)
03:47:54.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.348511 8884 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 4c24d4441f6c47728564eedff9749c8f [term 1 FOLLOWER]: Leader pre-election won for term 2
03:47:54.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.349066 8884 raft_consensus.cc:491] T 00000000000000000000000000000000 P 4c24d4441f6c47728564eedff9749c8f [term 1 FOLLOWER]: Starting leader election (detected failure of leader 59d0ebb2088d4fafbfc147165d7544f8)
03:47:54.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.349429 8884 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 4c24d4441f6c47728564eedff9749c8f [term 1 FOLLOWER]: Advancing to term 2
03:47:54.349 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.7.221.124:35655(127.7.221.124:35655)] connecting to peer
03:47:54.351 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.351 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.221.124:35655(127.7.221.124:35655)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.351 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.221.124:35655: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.353718 8884 raft_consensus.cc:513] T 00000000000000000000000000000000 P 4c24d4441f6c47728564eedff9749c8f [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c24d4441f6c47728564eedff9749c8f" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 43693 } } peers { permanent_uuid: "abe570d88d9e41929e679edcc29956c5" member_type: VOTER last_known_addr { host: "127.7.221.125" port: 33545 } } peers { permanent_uuid: "59d0ebb2088d4fafbfc147165d7544f8" member_type: VOTER last_known_addr { host: "127.7.221.124" port: 35655 } }
03:47:54.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.357266 8884 leader_election.cc:290] T 00000000000000000000000000000000 P 4c24d4441f6c47728564eedff9749c8f [CANDIDATE]: Term 2 election: Requested vote from peers abe570d88d9e41929e679edcc29956c5 (127.7.221.125:33545), 59d0ebb2088d4fafbfc147165d7544f8 (127.7.221.124:35655)
03:47:54.358 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655; will retry
03:47:54.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.360667 8885 leader_election.cc:290] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 4c24d4441f6c47728564eedff9749c8f (127.7.221.126:43693), 59d0ebb2088d4fafbfc147165d7544f8 (127.7.221.124:35655)
03:47:54.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.361171 8885 raft_consensus.cc:2802] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 [term 1 FOLLOWER]: Leader pre-election won for term 2
03:47:54.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.361416 8885 raft_consensus.cc:491] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 59d0ebb2088d4fafbfc147165d7544f8)
03:47:54.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.362600 8885 raft_consensus.cc:3058] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 [term 1 FOLLOWER]: Advancing to term 2
03:47:54.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.363646 8186 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4c24d4441f6c47728564eedff9749c8f" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "abe570d88d9e41929e679edcc29956c5"
03:47:54.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:54.369885 8080 leader_election.cc:336] T 00000000000000000000000000000000 P 4c24d4441f6c47728564eedff9749c8f [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 59d0ebb2088d4fafbfc147165d7544f8 (127.7.221.124:35655): Network error: Client connection negotiation failed: client connection to 127.7.221.124:35655: connect: Connection refused (error 111)
03:47:54.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.369947 8885 raft_consensus.cc:513] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c24d4441f6c47728564eedff9749c8f" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 43693 } } peers { permanent_uuid: "abe570d88d9e41929e679edcc29956c5" member_type: VOTER last_known_addr { host: "127.7.221.125" port: 33545 } } peers { permanent_uuid: "59d0ebb2088d4fafbfc147165d7544f8" member_type: VOTER last_known_addr { host: "127.7.221.124" port: 35655 } }
03:47:54.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.370522 8186 raft_consensus.cc:2391] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 4c24d4441f6c47728564eedff9749c8f in current term 2: Already voted for candidate abe570d88d9e41929e679edcc29956c5 in this term.
03:47:54.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:54.371095 8080 leader_election.cc:336] T 00000000000000000000000000000000 P 4c24d4441f6c47728564eedff9749c8f [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 59d0ebb2088d4fafbfc147165d7544f8 (127.7.221.124:35655): Network error: Client connection negotiation failed: client connection to 127.7.221.124:35655: connect: Connection refused (error 111)
03:47:54.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.371474 8083 leader_election.cc:304] T 00000000000000000000000000000000 P 4c24d4441f6c47728564eedff9749c8f [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4c24d4441f6c47728564eedff9749c8f; no voters: 59d0ebb2088d4fafbfc147165d7544f8, abe570d88d9e41929e679edcc29956c5
03:47:54.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.371843 8889 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 4c24d4441f6c47728564eedff9749c8f [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
03:47:54.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.372123 8106 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "abe570d88d9e41929e679edcc29956c5" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "4c24d4441f6c47728564eedff9749c8f"
03:47:54.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.372418 8106 raft_consensus.cc:2391] T 00000000000000000000000000000000 P 4c24d4441f6c47728564eedff9749c8f [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate abe570d88d9e41929e679edcc29956c5 in current term 2: Already voted for candidate 4c24d4441f6c47728564eedff9749c8f in this term.
03:47:54.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:54.372498 8161 leader_election.cc:336] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 59d0ebb2088d4fafbfc147165d7544f8 (127.7.221.124:35655): Network error: Client connection negotiation failed: client connection to 127.7.221.124:35655: connect: Connection refused (error 111)
03:47:54.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.373278 8885 leader_election.cc:290] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 [CANDIDATE]: Term 2 election: Requested vote from peers 4c24d4441f6c47728564eedff9749c8f (127.7.221.126:43693), 59d0ebb2088d4fafbfc147165d7544f8 (127.7.221.124:35655)
03:47:54.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:54.375119 8161 leader_election.cc:336] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 59d0ebb2088d4fafbfc147165d7544f8 (127.7.221.124:35655): Network error: Client connection negotiation failed: client connection to 127.7.221.124:35655: connect: Connection refused (error 111)
03:47:54.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.375267 8161 leader_election.cc:304] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: abe570d88d9e41929e679edcc29956c5; no voters: 4c24d4441f6c47728564eedff9749c8f, 59d0ebb2088d4fafbfc147165d7544f8
03:47:54.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.376056 8885 raft_consensus.cc:2747] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
03:47:54.389 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.7.221.124:35655(127.7.221.124:35655)] connecting to peer
03:47:54.390 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.390 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.221.124:35655(127.7.221.124:35655)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.391 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.221.124:35655: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.392 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655; will retry
03:47:54.429 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.7.221.124:35655(127.7.221.124:35655)] connecting to peer
03:47:54.430 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.430 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.221.124:35655(127.7.221.124:35655)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.430 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.221.124:35655: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.431 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655; will retry
03:47:54.469 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.7.221.124:35655(127.7.221.124:35655)] connecting to peer
03:47:54.471 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.471 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.221.124:35655(127.7.221.124:35655)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.471 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.221.124:35655: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.472 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655; will retry
03:47:54.472 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] sending RPC to server master-127.7.221.124:35655, [7ms] received response from server master-127.7.221.124:35655: Network error: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [7ms] delaying RPC due to: Network error: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.126:43693, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.125:33545, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.124:35655, [22ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.125:33545: OK, [23ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.124:35655: Network error: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [23ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.126:43693: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.126:43693, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.125:33545, [42ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.125:33545: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.126:43693: OK, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.124:35655, [45ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.124:35655: Network error: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [45ms] delaying RPC due to: Service unavailable: Master config (127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.126:43693, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.125:33545, [61ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.126:43693: OK, [64ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.125:33545: OK, [65ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.124:35655, [67ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.124:35655: Network error: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [67ms] delaying RPC due to: Service unavailable: Master config (127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.126:43693, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.125:33545, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.124:35655, [102ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.124:35655: Network error: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [109ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.126:43693: OK, [109ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.125:33545: OK, [109ms] delaying RPC due to: Service unavailable: Master config (127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.126:43693, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.125:33545, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.124:35655, [141ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.126:43693: OK, [141ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.124:35655: Network error: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [143ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.125:33545: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.126:43693, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.125:33545, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.124:35655, [181ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.124:35655: Network error: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [182ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.125:33545: OK, [182ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.126:43693: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.126:43693, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.125:33545, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.124:35655, [222ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.125:33545: OK, [222ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.126:43693: OK, [222ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.124:35655: Network error: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [223ms] delaying RPC due to: Service unavailable: Master config (127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, deferred=Deferred@1592835330(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$120/585998101@421e067d -> (continuation of Deferred@1604840566 after retry RPC after error@2103798862) -> (continuation of Deferred@648463949 after retry RPC after error@339280138) -> (continuation of Deferred@1948093351 after retry RPC after error@583835254) -> (continuation of Deferred@843624652 after retry RPC after error@1847969502) -> (continuation of Deferred@1996302350 after retry RPC after error@805582729) -> (continuation of Deferred@1195196309 after retry RPC after error@133042842), errback=org.apache.kudu.client.KuduTransaction$$Lambda$121/978413659@18cce731 -> (continuation of Deferred@1604840566 after retry RPC after error@2103798862) -> (continuation of Deferred@648463949 after retry RPC after error@339280138) -> (continuation of Deferred@1948093351 after retry RPC after error@583835254) -> (continuation of Deferred@843624652 after retry RPC after error@1847969502) -> (continuation of Deferred@1996302350 after retry RPC after error@805582729) -> (continuation of Deferred@1195196309 after retry RPC after error@133042842)))
03:47:54.473 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] sending RPC to server master-127.7.221.124:35655, [7ms] received response from server master-127.7.221.124:35655: Network error: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [7ms] delaying RPC due to: Network error: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.126:43693, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.125:33545, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.124:35655, [22ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.125:33545: OK, [23ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.124:35655: Network error: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [23ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.126:43693: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.126:43693, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.125:33545, [42ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.125:33545: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.126:43693: OK, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.124:35655, [45ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.124:35655: Network error: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [45ms] delaying RPC due to: Service unavailable: Master config (127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.126:43693, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.125:33545, [61ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.126:43693: OK, [64ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.125:33545: OK, [65ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.124:35655, [67ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.124:35655: Network error: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [67ms] delaying RPC due to: Service unavailable: Master config (127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.126:43693, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.125:33545, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.124:35655, [102ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.124:35655: Network error: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [109ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.126:43693: OK, [109ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.125:33545: OK, [109ms] delaying RPC due to: Service unavailable: Master config (127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.126:43693, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.125:33545, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.124:35655, [141ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.126:43693: OK, [141ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.124:35655: Network error: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [143ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.125:33545: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.126:43693, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.125:33545, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.124:35655, [181ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.124:35655: Network error: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [182ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.125:33545: OK, [182ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.126:43693: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.126:43693, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.125:33545, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.124:35655, [222ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.125:33545: OK, [222ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.126:43693: OK, [222ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.124:35655: Network error: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [223ms] delaying RPC due to: Service unavailable: Master config (127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, deferred=Deferred@1592835330(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$120/585998101@421e067d -> (continuation of Deferred@1604840566 after retry RPC after error@2103798862) -> (continuation of Deferred@648463949 after retry RPC after error@339280138) -> (continuation of Deferred@1948093351 after retry RPC after error@583835254) -> (continuation of Deferred@843624652 after retry RPC after error@1847969502) -> (continuation of Deferred@1996302350 after retry RPC after error@805582729) -> (continuation of Deferred@1195196309 after retry RPC after error@133042842), errback=org.apache.kudu.client.KuduTransaction$$Lambda$121/978413659@18cce731 -> (continuation of Deferred@1604840566 after retry RPC after error@2103798862) -> (continuation of Deferred@648463949 after retry RPC after error@339280138) -> (continuation of Deferred@1948093351 after retry RPC after error@583835254) -> (continuation of Deferred@843624652 after retry RPC after error@1847969502) -> (continuation of Deferred@1996302350 after retry RPC after error@805582729) -> (continuation of Deferred@1195196309 after retry RPC after error@133042842)))
03:47:54.489 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.7.221.124:35655(127.7.221.124:35655)] connecting to peer
03:47:54.490 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.490 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.221.124:35655(127.7.221.124:35655)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.491 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.221.124:35655: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.491 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655; will retry
03:47:54.510 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.7.221.124:35655(127.7.221.124:35655)] connecting to peer
03:47:54.511 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.511 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.221.124:35655(127.7.221.124:35655)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.511 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.221.124:35655: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.511 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655; will retry
03:47:54.529 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.7.221.124:35655(127.7.221.124:35655)] connecting to peer
03:47:54.530 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.530 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.221.124:35655(127.7.221.124:35655)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.530 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.221.124:35655: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.531 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655; will retry
03:47:54.549 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.7.221.124:35655(127.7.221.124:35655)] connecting to peer
03:47:54.550 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.550 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.221.124:35655(127.7.221.124:35655)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.550 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.221.124:35655: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.551 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655; will retry
03:47:54.569 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.7.221.124:35655(127.7.221.124:35655)] connecting to peer
03:47:54.570 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.570 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.221.124:35655(127.7.221.124:35655)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.570 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.221.124:35655: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.571 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655; will retry
03:47:54.589 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.7.221.124:35655(127.7.221.124:35655)] connecting to peer
03:47:54.590 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.590 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.221.124:35655(127.7.221.124:35655)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.590 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.221.124:35655: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.591 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655; will retry
03:47:54.629 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.7.221.124:35655(127.7.221.124:35655)] connecting to peer
03:47:54.631 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.631 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.221.124:35655(127.7.221.124:35655)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.631 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.221.124:35655: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.631 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655; will retry
03:47:54.689 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.7.221.124:35655(127.7.221.124:35655)] connecting to peer
03:47:54.691 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.691 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.221.124:35655(127.7.221.124:35655)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.691 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.221.124:35655: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.692 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655; will retry
03:47:54.693 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=203), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.126:43693, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.125:33545, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.124:35655, [1ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.124:35655: Network error: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [2ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.125:33545: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.126:43693: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.126:43693, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.125:33545, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.124:35655, [22ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.125:33545: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.126:43693: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.124:35655: Network error: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [23ms] delaying RPC due to: Service unavailable: Master config (127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.126:43693, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.125:33545, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.124:35655, [41ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.124:35655: Network error: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [41ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.125:33545: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.126:43693: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.126:43693, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.125:33545, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.124:35655, [61ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.124:35655: Network error: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [61ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.126:43693: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.125:33545: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.126:43693, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.125:33545, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.124:35655, [81ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.124:35655: Network error: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [82ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.125:33545: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.126:43693: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.126:43693, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.125:33545, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.124:35655, [101ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.124:35655: Network error: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [102ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.126:43693: OK, [102ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.125:33545: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.126:43693, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.125:33545, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.124:35655, [141ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.125:33545: OK, [142ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.126:43693: OK, [142ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.124:35655: Network error: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [142ms] delaying RPC due to: Service unavailable: Master config (127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.126:43693, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.125:33545, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.124:35655, [202ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.126:43693: OK, [202ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.124:35655: Network error: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [203ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.125:33545: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, deferred=Deferred@1827243709(state=PENDING, result=null, callback=(continuation of Deferred@1787624880 after retry RPC after error@1487135315) -> (continuation of Deferred@1786691201 after retry RPC after error@1241645341) -> (continuation of Deferred@1206781072 after retry RPC after error@1623876) -> (continuation of Deferred@521010032 after retry RPC after error@1172114277) -> (continuation of Deferred@1914719690 after retry RPC after error@885294637) -> (continuation of Deferred@1155677630 after retry RPC after error@1774817380) -> (continuation of Deferred@435674762 after retry RPC after error@160618103), errback=(continuation of Deferred@1787624880 after retry RPC after error@1487135315) -> (continuation of Deferred@1786691201 after retry RPC after error@1241645341) -> (continuation of Deferred@1206781072 after retry RPC after error@1623876) -> (continuation of Deferred@521010032 after retry RPC after error@1172114277) -> (continuation of Deferred@1914719690 after retry RPC after error@885294637) -> (continuation of Deferred@1155677630 after retry RPC after error@1774817380) -> (continuation of Deferred@435674762 after retry RPC after error@160618103)))
03:47:54.693 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=203), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.126:43693, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.125:33545, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.124:35655, [1ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.124:35655: Network error: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [2ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.125:33545: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.126:43693: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.126:43693, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.125:33545, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.124:35655, [22ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.125:33545: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.126:43693: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.124:35655: Network error: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [23ms] delaying RPC due to: Service unavailable: Master config (127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.126:43693, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.125:33545, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.124:35655, [41ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.124:35655: Network error: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [41ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.125:33545: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.126:43693: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.126:43693, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.125:33545, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.124:35655, [61ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.124:35655: Network error: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [61ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.126:43693: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.125:33545: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.126:43693, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.125:33545, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.124:35655, [81ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.124:35655: Network error: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [82ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.125:33545: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.126:43693: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.126:43693, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.125:33545, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.124:35655, [101ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.124:35655: Network error: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [102ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.126:43693: OK, [102ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.125:33545: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.126:43693, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.125:33545, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.124:35655, [141ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.125:33545: OK, [142ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.126:43693: OK, [142ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.124:35655: Network error: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [142ms] delaying RPC due to: Service unavailable: Master config (127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.126:43693, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.125:33545, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.124:35655, [202ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.126:43693: OK, [202ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.124:35655: Network error: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [203ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.125:33545: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, deferred=Deferred@1827243709(state=PENDING, result=null, callback=(continuation of Deferred@1787624880 after retry RPC after error@1487135315) -> (continuation of Deferred@1786691201 after retry RPC after error@1241645341) -> (continuation of Deferred@1206781072 after retry RPC after error@1623876) -> (continuation of Deferred@521010032 after retry RPC after error@1172114277) -> (continuation of Deferred@1914719690 after retry RPC after error@885294637) -> (continuation of Deferred@1155677630 after retry RPC after error@1774817380) -> (continuation of Deferred@435674762 after retry RPC after error@160618103), errback=(continuation of Deferred@1787624880 after retry RPC after error@1487135315) -> (continuation of Deferred@1786691201 after retry RPC after error@1241645341) -> (continuation of Deferred@1206781072 after retry RPC after error@1623876) -> (continuation of Deferred@521010032 after retry RPC after error@1172114277) -> (continuation of Deferred@1914719690 after retry RPC after error@885294637) -> (continuation of Deferred@1155677630 after retry RPC after error@1774817380) -> (continuation of Deferred@435674762 after retry RPC after error@160618103)))
03:47:54.709 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.7.221.124:35655(127.7.221.124:35655)] connecting to peer
03:47:54.710 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.710 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.221.124:35655(127.7.221.124:35655)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.711 [INFO - pool-38-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.221.124:35655: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.711 [WARN - pool-38-thread-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655; will retry
03:47:54.729 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.7.221.124:35655(127.7.221.124:35655)] connecting to peer
03:47:54.730 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.730 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.221.124:35655(127.7.221.124:35655)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.730 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.221.124:35655: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.731 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655; will retry
03:47:54.749 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.7.221.124:35655(127.7.221.124:35655)] connecting to peer
03:47:54.750 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.750 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.221.124:35655(127.7.221.124:35655)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.751 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.221.124:35655: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.751 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655; will retry
03:47:54.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.752143 8885 raft_consensus.cc:491] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:47:54.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.752357 8885 raft_consensus.cc:513] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c24d4441f6c47728564eedff9749c8f" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 43693 } } peers { permanent_uuid: "abe570d88d9e41929e679edcc29956c5" member_type: VOTER last_known_addr { host: "127.7.221.125" port: 33545 } } peers { permanent_uuid: "59d0ebb2088d4fafbfc147165d7544f8" member_type: VOTER last_known_addr { host: "127.7.221.124" port: 35655 } }
03:47:54.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.753113 8885 leader_election.cc:290] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 4c24d4441f6c47728564eedff9749c8f (127.7.221.126:43693), 59d0ebb2088d4fafbfc147165d7544f8 (127.7.221.124:35655)
03:47:54.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.753752 8106 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "abe570d88d9e41929e679edcc29956c5" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "4c24d4441f6c47728564eedff9749c8f" is_pre_election: true
03:47:54.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.754040 8106 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 4c24d4441f6c47728564eedff9749c8f [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate abe570d88d9e41929e679edcc29956c5 in term 2.
03:47:54.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.754540 8164 leader_election.cc:304] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4c24d4441f6c47728564eedff9749c8f, abe570d88d9e41929e679edcc29956c5; no voters:
03:47:54.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:54.754730 8161 leader_election.cc:336] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 59d0ebb2088d4fafbfc147165d7544f8 (127.7.221.124:35655): Network error: Client connection negotiation failed: client connection to 127.7.221.124:35655: connect: Connection refused (error 111)
03:47:54.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.754900 8885 raft_consensus.cc:2802] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 [term 2 FOLLOWER]: Leader pre-election won for term 3
03:47:54.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.754999 8885 raft_consensus.cc:491] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:47:54.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.755123 8885 raft_consensus.cc:3058] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 [term 2 FOLLOWER]: Advancing to term 3
03:47:54.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.757195 8885 raft_consensus.cc:513] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c24d4441f6c47728564eedff9749c8f" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 43693 } } peers { permanent_uuid: "abe570d88d9e41929e679edcc29956c5" member_type: VOTER last_known_addr { host: "127.7.221.125" port: 33545 } } peers { permanent_uuid: "59d0ebb2088d4fafbfc147165d7544f8" member_type: VOTER last_known_addr { host: "127.7.221.124" port: 35655 } }
03:47:54.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.757906 8885 leader_election.cc:290] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 [CANDIDATE]: Term 3 election: Requested vote from peers 4c24d4441f6c47728564eedff9749c8f (127.7.221.126:43693), 59d0ebb2088d4fafbfc147165d7544f8 (127.7.221.124:35655)
03:47:54.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.758447 8106 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "abe570d88d9e41929e679edcc29956c5" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "4c24d4441f6c47728564eedff9749c8f"
03:47:54.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.758652 8106 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 4c24d4441f6c47728564eedff9749c8f [term 2 FOLLOWER]: Advancing to term 3
03:47:54.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:54.759424 8161 leader_election.cc:336] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 59d0ebb2088d4fafbfc147165d7544f8 (127.7.221.124:35655): Network error: Client connection negotiation failed: client connection to 127.7.221.124:35655: connect: Connection refused (error 111)
03:47:54.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.760785 8106 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 4c24d4441f6c47728564eedff9749c8f [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate abe570d88d9e41929e679edcc29956c5 in term 3.
03:47:54.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.761293 8164 leader_election.cc:304] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 4c24d4441f6c47728564eedff9749c8f, abe570d88d9e41929e679edcc29956c5; no voters: 59d0ebb2088d4fafbfc147165d7544f8
03:47:54.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.761581 8885 raft_consensus.cc:2802] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 [term 3 FOLLOWER]: Leader election won for term 3
03:47:54.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.761758 8885 raft_consensus.cc:695] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 [term 3 LEADER]: Becoming Leader. State: Replica: abe570d88d9e41929e679edcc29956c5, State: Running, Role: LEADER
03:47:54.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.762203 8885 consensus_queue.cc:237] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c24d4441f6c47728564eedff9749c8f" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 43693 } } peers { permanent_uuid: "abe570d88d9e41929e679edcc29956c5" member_type: VOTER last_known_addr { host: "127.7.221.125" port: 33545 } } peers { permanent_uuid: "59d0ebb2088d4fafbfc147165d7544f8" member_type: VOTER last_known_addr { host: "127.7.221.124" port: 35655 } }
03:47:54.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.763932 8888 sys_catalog.cc:455] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 [sys.catalog]: SysCatalogTable state changed. Reason: New leader abe570d88d9e41929e679edcc29956c5. Latest consensus state: current_term: 3 leader_uuid: "abe570d88d9e41929e679edcc29956c5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c24d4441f6c47728564eedff9749c8f" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 43693 } } peers { permanent_uuid: "abe570d88d9e41929e679edcc29956c5" member_type: VOTER last_known_addr { host: "127.7.221.125" port: 33545 } } peers { permanent_uuid: "59d0ebb2088d4fafbfc147165d7544f8" member_type: VOTER last_known_addr { host: "127.7.221.124" port: 35655 } } }
03:47:54.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.764288 8888 sys_catalog.cc:458] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 [sys.catalog]: This master's current role is: LEADER
03:47:54.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.765033 8896 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
03:47:54.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.767661 8896 catalog_manager.cc:671] Loaded metadata for table txn_manager_fallback [id=1f8120a6dc0743938fb8db3498d07cae]
03:47:54.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.768168 8896 catalog_manager.cc:671] Loaded metadata for table kudu_system.kudu_transactions [id=b19531f9f14141c7b6e8abf776a43683]
03:47:54.769 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.7.221.124:35655(127.7.221.124:35655)] connecting to peer
03:47:54.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.770370 8896 tablet_loader.cc:96] loaded metadata for tablet 729fc2d921ee42a0b407222da665b01c (table txn_manager_fallback [id=1f8120a6dc0743938fb8db3498d07cae])
03:47:54.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.770928 8896 tablet_loader.cc:96] loaded metadata for tablet 84d3092f319b4ee2808165b03e84a2fe (table kudu_system.kudu_transactions [id=b19531f9f14141c7b6e8abf776a43683])
03:47:54.771 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.771404 8896 tablet_loader.cc:96] loaded metadata for tablet f35afbde3ffe42e9aa2a447b50aed40c (table txn_manager_fallback [id=1f8120a6dc0743938fb8db3498d07cae])
03:47:54.771 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.221.124:35655(127.7.221.124:35655)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.771752 8896 catalog_manager.cc:1486] Initializing Kudu cluster ID...
03:47:54.771 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.221.124:35655: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.772 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655; will retry
03:47:54.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.773080 8896 catalog_manager.cc:1261] Loaded cluster ID: 01957b9e343a47d7bd313a67674c48c6
03:47:54.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.773180 8896 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
03:47:54.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.774411 8896 catalog_manager.cc:1506] Loading token signing keys...
03:47:54.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.775303 8896 catalog_manager.cc:5966] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5: Loaded TSK: 0
03:47:54.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.776201 8896 catalog_manager.cc:1516] Initializing in-progress tserver states...
03:47:54.789 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.7.221.124:35655(127.7.221.124:35655)] connecting to peer
03:47:54.790 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.790 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.221.124:35655(127.7.221.124:35655)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.790 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.221.124:35655: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655
03:47:54.792 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
03:47:54.793 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=5, TimeoutTracker(timeout=250, elapsed=83), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.126:43693, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.125:33545, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.124:35655, [2ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.125:33545: OK, [3ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.124:35655: Network error: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [3ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.126:43693: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.126:43693, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.125:33545, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.124:35655, [22ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.124:35655: Network error: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [22ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.126:43693: OK, [23ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.125:33545: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.126:43693, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.125:33545, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.124:35655, [42ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.124:35655: Network error: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [43ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.125:33545: OK, [43ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.126:43693: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.126:43693, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.125:33545, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.124:35655, [63ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.126:43693: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.124:35655: Network error: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [64ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.125:33545: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.7.221.126:43693,127.7.221.125:33545,127.7.221.124:35655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.126:43693, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.125:33545, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.221.124:35655, [82ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.124:35655: Network error: Failed to connect to peer master-127.7.221.124:35655(127.7.221.124:35655): Connection refused: /127.7.221.124:35655, [83ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.126:43693: OK, [84ms] Sub RPC ConnectToMaster: received response from server master-127.7.221.125:33545: OK, deferred=Deferred@1572924966(state=PENDING, result=null, callback=(continuation of Deferred@453172697 after retry RPC after error@1584845132) -> (continuation of Deferred@1987526962 after retry RPC after error@748433412) -> (continuation of Deferred@570285147 after retry RPC after error@1240044953) -> (continuation of Deferred@98247495 after retry RPC after error@1217019880), errback=(continuation of Deferred@453172697 after retry RPC after error@1584845132) -> (continuation of Deferred@1987526962 after retry RPC after error@748433412) -> (continuation of Deferred@570285147 after retry RPC after error@1240044953) -> (continuation of Deferred@98247495 after retry RPC after error@1217019880))) after lookup
03:47:54.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.847465 8176 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:34686:
03:47:54.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
03:47:54.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
03:47:54.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:47:54.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
03:47:54.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
03:47:54.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
03:47:54.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:47:54.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
03:47:54.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
03:47:54.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:47:54.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:47:54.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:47:54.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:47:54.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
03:47:54.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
03:47:54.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
03:47:54.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:47:54.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
03:47:54.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
03:47:54.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:47:54.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:47:54.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:47:54.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:47:54.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
03:47:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
03:47:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
03:47:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:47:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
03:47:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
03:47:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:47:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:47:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:47:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:47:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
03:47:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
03:47:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
03:47:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:47:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
03:47:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
03:47:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:47:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:47:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:47:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:47:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
03:47:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
03:47:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
03:47:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
03:47:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:47:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
03:47:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
03:47:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:47:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
03:47:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:47:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:47:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:47:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
03:47:54.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:54.853602 8176 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
03:47:54.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:54.854028 8161 consensus_peers.cc:487] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 -> Peer 59d0ebb2088d4fafbfc147165d7544f8 (127.7.221.124:35655): Couldn't send request to peer 59d0ebb2088d4fafbfc147165d7544f8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.124:35655: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
03:47:54.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.860222 8106 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 4c24d4441f6c47728564eedff9749c8f [term 3 FOLLOWER]: Refusing update from remote peer abe570d88d9e41929e679edcc29956c5: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 3 index: 12. (index mismatch)
03:47:54.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.861033 8888 consensus_queue.cc:1035] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c24d4441f6c47728564eedff9749c8f" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 43693 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
03:47:54.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.864534 8889 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4c24d4441f6c47728564eedff9749c8f [sys.catalog]: SysCatalogTable state changed. Reason: New leader abe570d88d9e41929e679edcc29956c5. Latest consensus state: current_term: 3 leader_uuid: "abe570d88d9e41929e679edcc29956c5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c24d4441f6c47728564eedff9749c8f" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 43693 } } peers { permanent_uuid: "abe570d88d9e41929e679edcc29956c5" member_type: VOTER last_known_addr { host: "127.7.221.125" port: 33545 } } peers { permanent_uuid: "59d0ebb2088d4fafbfc147165d7544f8" member_type: VOTER last_known_addr { host: "127.7.221.124" port: 35655 } } }
03:47:54.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.864929 8889 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4c24d4441f6c47728564eedff9749c8f [sys.catalog]: This master's current role is: FOLLOWER
03:47:54.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.878754 8885 sys_catalog.cc:455] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "abe570d88d9e41929e679edcc29956c5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c24d4441f6c47728564eedff9749c8f" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 43693 } } peers { permanent_uuid: "abe570d88d9e41929e679edcc29956c5" member_type: VOTER last_known_addr { host: "127.7.221.125" port: 33545 } } peers { permanent_uuid: "59d0ebb2088d4fafbfc147165d7544f8" member_type: VOTER last_known_addr { host: "127.7.221.124" port: 35655 } } }
03:47:54.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.879086 8885 sys_catalog.cc:458] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 [sys.catalog]: This master's current role is: LEADER
03:47:54.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.925829 8386 txn_status_manager.cc:1391] automatically aborted stale txn (ID 0) past 1.134s from last keepalive heartbeat (effective timeout is 1.000s)
03:47:54.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:54.926301 8208 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:34660 (request call id 21) took 131 ms (client timeout 165 ms). Trace:
03:47:54.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:54.926443 8208 rpcz_store.cc:269] 0625 03:47:54.794566 (+ 0us) service_pool.cc:168] Inserting onto call queue
03:47:54.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0625 03:47:54.794660 (+ 94us) service_pool.cc:225] Handling call
03:47:54.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0625 03:47:54.926217 (+131557us) inbound_call.cc:173] Queueing success response
03:47:54.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"dns_us":699,"init.queue_time_us":631,"init.run_cpu_time_us":9807,"init.run_wall_time_us":63298,"thread_start_us":1414,"threads_started":5}
03:47:54.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:54.927747 8428 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.7.221.124:35655)
03:47:54.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:54.934504 8911 tablet_replica.cc:1304] Aborted: operation has been aborted: cancelling pending write operations
03:47:54.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.943634 8889 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4c24d4441f6c47728564eedff9749c8f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "abe570d88d9e41929e679edcc29956c5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c24d4441f6c47728564eedff9749c8f" member_type: VOTER last_known_addr { host: "127.7.221.126" port: 43693 } } peers { permanent_uuid: "abe570d88d9e41929e679edcc29956c5" member_type: VOTER last_known_addr { host: "127.7.221.125" port: 33545 } } peers { permanent_uuid: "59d0ebb2088d4fafbfc147165d7544f8" member_type: VOTER last_known_addr { host: "127.7.221.124" port: 35655 } } }
03:47:54.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:54.944020 8889 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4c24d4441f6c47728564eedff9749c8f [sys.catalog]: This master's current role is: FOLLOWER
03:47:55.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:55.149825 8161 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.221.124:35655: connect: Connection refused (error 111) [suppressed 11 similar messages]
03:47:55.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:55.364919 8161 consensus_peers.cc:487] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 -> Peer 59d0ebb2088d4fafbfc147165d7544f8 (127.7.221.124:35655): Couldn't send request to peer 59d0ebb2088d4fafbfc147165d7544f8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.124:35655: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
03:47:55.474 [DEBUG - kudu-nio-1] (KuduTransaction.java:799) terminating keepalive task (txn ID 0) due to exception org.apache.kudu.client.NonRecoverableException: Failed to write to server: ef30ee403f0246569128f239c8802ffb (127.7.221.65:34863): transaction ID 0 is not available for further commits: state: ABORTED user: "slave" start_timestamp: 1750823273 last_transition_timestamp: 1750823274
03:47:55.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:55.822089 8161 consensus_peers.cc:487] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 -> Peer 59d0ebb2088d4fafbfc147165d7544f8 (127.7.221.124:35655): Couldn't send request to peer 59d0ebb2088d4fafbfc147165d7544f8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.124:35655: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
03:47:55.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:55.861131 8065 master_runner.cc:186] Error getting master registration for 127.7.221.124:35655: Network error: Client connection negotiation failed: client connection to 127.7.221.124:35655: connect: Connection refused (error 111),
03:47:55.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:55.861322 8065 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:47:55.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:55.910849 8800 heartbeater.cc:499] Master 127.7.221.125:33545 was elected leader, sending a full tablet report...
03:47:55.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:55.919998 8655 heartbeater.cc:499] Master 127.7.221.125:33545 was elected leader, sending a full tablet report...
03:47:56.068 [DEBUG - main] (Connection.java:575) [peer master-127.7.221.126:43693(127.7.221.126:43693)] disconnecting while in state READY
03:47:56.068 [DEBUG - main] (Connection.java:575) [peer ef30ee403f0246569128f239c8802ffb(127.7.221.65:34863)] disconnecting while in state READY
03:47:56.068 [DEBUG - main] (Connection.java:575) [peer master-127.7.221.124:35655(127.7.221.124:35655)] disconnecting while in state TERMINATED
03:47:56.068 [DEBUG - main] (Connection.java:575) [peer master-127.7.221.125:33545(127.7.221.125:33545)] disconnecting while in state READY
03:47:56.069 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.7.221.126:43693(127.7.221.126:43693)] handling channelInactive
03:47:56.069 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.221.126:43693(127.7.221.126:43693)] cleaning up while in state READY due to: connection closed
03:47:56.069 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
03:47:56.070 [DEBUG - kudu-nio-1] (Connection.java:239) [peer ef30ee403f0246569128f239c8802ffb(127.7.221.65:34863)] handling channelInactive
03:47:56.070 [DEBUG - kudu-nio-1] (Connection.java:709) [peer ef30ee403f0246569128f239c8802ffb(127.7.221.65:34863)] cleaning up while in state READY due to: connection closed
03:47:56.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:56.070614 8053 external_mini_cluster.cc:1661] Attempting to check leaks for /tmp/dist-test-taskqbWxBn/build/asan/bin/kudu pid 8364
03:47:56.070 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.7.221.125:33545(127.7.221.125:33545)] handling channelInactive
03:47:56.070 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.221.125:33545(127.7.221.125:33545)] cleaning up while in state READY due to: connection closed
03:47:56.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:56.083271 8501 generic_service.cc:196] Checking for leaks (request via RPC)
03:47:56.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:56.215910 8053 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskqbWxBn/build/asan/bin/kudu with pid 8364
03:47:56.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:56.238744 8673 connection.cc:537] server connection from 127.7.221.65:38717 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
03:47:56.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:56.239403 8053 external_mini_cluster.cc:1661] Attempting to check leaks for /tmp/dist-test-taskqbWxBn/build/asan/bin/kudu pid 8511
03:47:56.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:56.247390 8648 generic_service.cc:196] Checking for leaks (request via RPC)
03:47:56.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:47:56.321800 8161 consensus_peers.cc:487] T 00000000000000000000000000000000 P abe570d88d9e41929e679edcc29956c5 -> Peer 59d0ebb2088d4fafbfc147165d7544f8 (127.7.221.124:35655): Couldn't send request to peer 59d0ebb2088d4fafbfc147165d7544f8. Status: Network error: Client connection negotiation failed: client connection to 127.7.221.124:35655: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
03:47:56.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:56.343683 8053 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskqbWxBn/build/asan/bin/kudu with pid 8511
03:47:56.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:56.361704 8053 external_mini_cluster.cc:1661] Attempting to check leaks for /tmp/dist-test-taskqbWxBn/build/asan/bin/kudu pid 8656
03:47:56.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:56.369153 8793 generic_service.cc:196] Checking for leaks (request via RPC)
03:47:56.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:56.465775 8053 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskqbWxBn/build/asan/bin/kudu with pid 8656
03:47:56.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:56.483829 8053 external_mini_cluster.cc:1661] Attempting to check leaks for /tmp/dist-test-taskqbWxBn/build/asan/bin/kudu pid 8065
03:47:56.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:56.485148 8136 generic_service.cc:196] Checking for leaks (request via RPC)
03:47:56.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:56.599840 8053 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskqbWxBn/build/asan/bin/kudu with pid 8065
03:47:56.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:56.621678 8053 external_mini_cluster.cc:1661] Attempting to check leaks for /tmp/dist-test-taskqbWxBn/build/asan/bin/kudu pid 8143
03:47:56.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:56.622848 8218 generic_service.cc:196] Checking for leaks (request via RPC)
03:47:56.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:47:56.741272 8053 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskqbWxBn/build/asan/bin/kudu with pid 8143
03:47:56.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-06-25T03:47:56Z chronyd exiting
03:47:56.893 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveSwitchesToOtherTxnManager: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Failed to write to server: ef30ee403f0246569128f239c8802ffb (127.7.221.65:34863): transaction ID 0 is not open: state: ABORTED user: "slave" start_timestamp: 1750823273 last_transition_timestamp: 1750823274
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduTransaction.doCommitTransaction(KuduTransaction.java:491) ~[main/:?]
at org.apache.kudu.client.KuduTransaction.commitWithMode(KuduTransaction.java:537) ~[main/:?]
at org.apache.kudu.client.KuduTransaction.commit(KuduTransaction.java:284) ~[main/:?]
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveSwitchesToOtherTxnManager(TestKuduTransaction.java:1152) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.RpcProxy.dispatchTxnManagerError(RpcProxy.java:439) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:298) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1503) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1366) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1415) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Failed to write to server: ef30ee403f0246569128f239c8802ffb (127.7.221.65:34863): transaction ID 0 is not open: state: ABORTED user: "slave" start_timestamp: 1750823273 last_transition_timestamp: 1750823274
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduTransaction.doCommitTransaction(KuduTransaction.java:491)
at org.apache.kudu.client.KuduTransaction.commitWithMode(KuduTransaction.java:537)
at org.apache.kudu.client.KuduTransaction.commit(KuduTransaction.java:284)
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveSwitchesToOtherTxnManager(TestKuduTransaction.java:1152)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.RpcProxy.dispatchTxnManagerError(RpcProxy.java:439)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:298)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1503)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1366)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1415)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more