08:03:33.816 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
08:03:33.817 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task8V1MUJ/test-tmp
08:03:33.817 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/asan/bin
08:03:33.817 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task8V1MUJ/java/build/dist-test/../../../build/asan/bin/kudu, test, mini_cluster, --serialization=pb]
08:03:33.818 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022"
extra_master_flags: "--txn_manager_enabled"
extra_master_flags: "--raft_heartbeat_interval_ms=100"
extra_tserver_flags: "--txn_keepalive_interval_ms=1000"
extra_tserver_flags: "--txn_staleness_tracker_interval_ms=250"
extra_tserver_flags: "--enable_txn_system_client_init=true"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
08:03:34.123 [DEBUG - main] (MiniKuduCluster.java:195) Response:
08:03:34.123 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
08:03:34.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.125376 6591 mini_oidc.cc:150] Starting JWKS server
08:03:34.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.127750 6591 webserver.cc:469] Webserver started at http://127.0.0.1:40499/ using document root <none> and password file <none>
08:03:34.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.128027 6591 mini_oidc.cc:186] Starting OIDC Discovery server
08:03:34.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.128602 6591 webserver.cc:469] Webserver started at http://127.0.0.1:45085/ using document root <none> and password file <none>
08:03:34.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-06-25T08:03:34Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
08:03:34.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-06-25T08:03:34Z Disabled control of system clock
08:03:34.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.157531 6591 external_mini_cluster.cc:1351] Running /tmp/dist-test-task8V1MUJ/build/asan/bin/kudu
08:03:34.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8V1MUJ/build/asan/bin/kudu
08:03:34.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/master-0/wal
08:03:34.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/master-0/data
08:03:34.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:34.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:34.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:34.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:34.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:34.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:34.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/master-0/logs
08:03:34.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/master-0/data/info.pb
08:03:34.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:34.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:34.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:34.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:34.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:34.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:34.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:34.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:34.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:03:34.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:34.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:03:34.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:03:34.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.111.254:46321
08:03:34.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.111.254
08:03:34.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:34.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.111.212:42157
08:03:34.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:34.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:34.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:34.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:03:34.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.111.254:46321,127.6.111.253:39259,127.6.111.252:40087
08:03:34.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:03:34.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40499/jwks.json
08:03:34.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
08:03:34.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
08:03:34.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:34.457866 6602 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:03:34.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:34.458158 6602 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:34.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:34.458235 6602 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:03:34.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:34.465798 6602 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:03:34.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:34.465876 6602 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
08:03:34.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:34.465924 6602 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:03:34.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:34.465965 6602 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:03:34.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:34.466015 6602 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:03:34.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:34.466053 6602 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:03:34.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:34.466104 6602 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40499/jwks.json
08:03:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.474970 6602 master_runner.cc:386] Master server non-default flags:
08:03:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.111.212:42157
08:03:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
08:03:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/master-0/data
08:03:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/master-0/wal
08:03:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:03:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.111.254:46321,127.6.111.253:39259,127.6.111.252:40087
08:03:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
08:03:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:03:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.111.254:46321
08:03:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:03:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:03:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40499/jwks.json
08:03:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/master-0/data/info.pb
08:03:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.111.254
08:03:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/master-0/logs
08:03:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:03:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:03:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:03:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6d72d4a242076468501f3430b9a2cd050c634be2
08:03:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:03:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Jun 2025 07:43:22 UTC on 24a791456cd2
08:03:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6760
08:03:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
08:03:34.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.476418 6602 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:34.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.478111 6602 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:34.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:34.487879 6608 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:34.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:34.487933 6610 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:34.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:34.487967 6607 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:34.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.488917 6602 server_base.cc:1048] running on GCE node
08:03:34.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.489609 6602 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:34.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.490942 6602 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.492185 6602 hybrid_clock.cc:648] HybridClock initialized: now 1750838614492114 us; error 52 us; skew 500 ppm
08:03:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.492473 6602 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:03:34.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.494650 6602 webserver.cc:469] Webserver started at http://127.6.111.254:39873/ using document root <none> and password file <none>
08:03:34.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.495275 6602 fs_manager.cc:362] Metadata directory not provided
08:03:34.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.495370 6602 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:34.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.495656 6602 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
08:03:34.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.497927 6602 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/master-0/data/instance:
08:03:34.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "37df082dc4d74885bb14a018ac6c52a1"
08:03:34.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 08:03:34 on dist-test-slave-gp61"
08:03:34.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.498651 6602 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/master-0/wal/instance:
08:03:34.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "37df082dc4d74885bb14a018ac6c52a1"
08:03:34.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 08:03:34 on dist-test-slave-gp61"
08:03:34.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.503597 6602 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.002s sys 0.003s
08:03:34.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.506541 6616 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:03:34.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.507692 6602 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
08:03:34.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.507827 6602 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/master-0/data,/tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/master-0/wal
08:03:34.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "37df082dc4d74885bb14a018ac6c52a1"
08:03:34.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 08:03:34 on dist-test-slave-gp61"
08:03:34.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.507968 6602 fs_report.cc:389] FS layout report
08:03:34.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:34.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/master-0/wal
08:03:34.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/master-0/wal
08:03:34.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/master-0/data/data
08:03:34.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:34.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:34.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:34.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:34.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:34.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:34.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:34.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:34.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:34.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:34.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:34.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.538340 6602 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:03:34.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.539165 6602 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:34.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.539405 6602 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:34.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.562392 6602 rpc_server.cc:307] RPC server started. Bound to: 127.6.111.254:46321
08:03:34.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.562433 6677 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.6.111.254:46321 every 8 connection(s)
08:03:34.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.563886 6602 server_base.cc:1180] Dumped server information to /tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/master-0/data/info.pb
08:03:34.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.567579 6678 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:34.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.573089 6591 external_mini_cluster.cc:1413] Started /tmp/dist-test-task8V1MUJ/build/asan/bin/kudu as pid 6602
08:03:34.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.573450 6591 external_mini_cluster.cc:1427] Reading /tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/master-0/wal/instance
08:03:34.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.573702 6678 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.111.254" port: 46321 } has no permanent_uuid. Determining permanent_uuid...
08:03:34.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.576308 6591 external_mini_cluster.cc:1351] Running /tmp/dist-test-task8V1MUJ/build/asan/bin/kudu
08:03:34.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8V1MUJ/build/asan/bin/kudu
08:03:34.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/master-1/wal
08:03:34.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/master-1/data
08:03:34.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:34.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:34.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:34.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:34.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:34.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:34.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/master-1/logs
08:03:34.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/master-1/data/info.pb
08:03:34.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:34.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:34.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:34.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:34.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:34.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:34.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:34.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:34.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:03:34.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:34.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:03:34.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:03:34.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.111.253:39259
08:03:34.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.111.253
08:03:34.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:34.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.111.212:42157
08:03:34.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:34.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:34.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:34.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:03:34.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.111.254:46321,127.6.111.253:39259,127.6.111.252:40087
08:03:34.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:03:34.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40499/jwks.json
08:03:34.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
08:03:34.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
08:03:34.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.592494 6678 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.111.253" port: 39259 } has no permanent_uuid. Determining permanent_uuid...
08:03:34.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:34.593640 6620 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.111.253:39259: connect: Connection refused (error 111)
08:03:34.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:34.595218 6678 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.111.253:39259: Network error: Client connection negotiation failed: client connection to 127.6.111.253:39259: connect: Connection refused (error 111)
08:03:34.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.644593 6678 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.111.253" port: 39259 } attempt: 1
08:03:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:34.646708 6678 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.111.253:39259: Network error: Client connection negotiation failed: client connection to 127.6.111.253:39259: connect: Connection refused (error 111)
08:03:34.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.715015 6678 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.111.253" port: 39259 } attempt: 2
08:03:34.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:34.716988 6678 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.111.253:39259: Network error: Client connection negotiation failed: client connection to 127.6.111.253:39259: connect: Connection refused (error 111)
08:03:34.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.808277 6678 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.111.253" port: 39259 } attempt: 3
08:03:34.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:34.810482 6678 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.111.253:39259: Network error: Client connection negotiation failed: client connection to 127.6.111.253:39259: connect: Connection refused (error 111)
08:03:34.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:34.890034 6682 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:03:34.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:34.890379 6682 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:34.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:34.890497 6682 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:03:34.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:34.898607 6682 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:03:34.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:34.898728 6682 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
08:03:34.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:34.898814 6682 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:03:34.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:34.898872 6682 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:03:34.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:34.898953 6682 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:03:34.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:34.899019 6682 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:03:34.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:34.899088 6682 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40499/jwks.json
08:03:34.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.908875 6682 master_runner.cc:386] Master server non-default flags:
08:03:34.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:34.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.111.212:42157
08:03:34.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:34.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:34.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
08:03:34.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/master-1/data
08:03:34.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/master-1/wal
08:03:34.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:03:34.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.111.254:46321,127.6.111.253:39259,127.6.111.252:40087
08:03:34.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
08:03:34.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:34.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:34.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:03:34.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.111.253:39259
08:03:34.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:03:34.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:34.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:03:34.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40499/jwks.json
08:03:34.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:34.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:34.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/master-1/data/info.pb
08:03:34.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.111.253
08:03:34.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:34.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:34.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:34.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:34.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:34.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:34.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/master-1/logs
08:03:34.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:34.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:34.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:03:34.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:03:34.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:03:34.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6d72d4a242076468501f3430b9a2cd050c634be2
08:03:34.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:03:34.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Jun 2025 07:43:22 UTC on 24a791456cd2
08:03:34.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6760
08:03:34.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
08:03:34.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.910856 6682 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:34.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.912868 6682 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:34.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:34.922712 6688 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:34.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:34.922814 6689 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:34.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:34.923367 6691 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:34.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.923779 6682 server_base.cc:1048] running on GCE node
08:03:34.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.924482 6682 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:34.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.925779 6682 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:34.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.926999 6682 hybrid_clock.cc:648] HybridClock initialized: now 1750838614926944 us; error 31 us; skew 500 ppm
08:03:34.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.927342 6682 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:03:34.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.929615 6682 webserver.cc:469] Webserver started at http://127.6.111.253:32801/ using document root <none> and password file <none>
08:03:34.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.930298 6682 fs_manager.cc:362] Metadata directory not provided
08:03:34.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.930420 6682 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:34.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.930755 6682 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
08:03:34.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.933104 6682 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/master-1/data/instance:
08:03:34.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "98a4571016524600ae1e653f157c03c0"
08:03:34.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 08:03:34 on dist-test-slave-gp61"
08:03:34.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.933856 6682 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/master-1/wal/instance:
08:03:34.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "98a4571016524600ae1e653f157c03c0"
08:03:34.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 08:03:34 on dist-test-slave-gp61"
08:03:34.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.938639 6682 fs_manager.cc:696] Time spent creating directory manager: real 0.004s user 0.005s sys 0.000s
08:03:34.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.941730 6697 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:03:34.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.942942 6682 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.001s
08:03:34.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.943112 6682 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/master-1/data,/tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/master-1/wal
08:03:34.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "98a4571016524600ae1e653f157c03c0"
08:03:34.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 08:03:34 on dist-test-slave-gp61"
08:03:34.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.943286 6682 fs_report.cc:389] FS layout report
08:03:34.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:34.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/master-1/wal
08:03:34.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/master-1/wal
08:03:34.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/master-1/data/data
08:03:34.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:34.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:34.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:34.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:34.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:34.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:34.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:34.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:34.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:34.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:34.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:34.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.953711 6678 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.111.253" port: 39259 } attempt: 4
08:03:34.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:34.955418 6678 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.111.253:39259: Network error: Client connection negotiation failed: client connection to 127.6.111.253:39259: connect: Connection refused (error 111)
08:03:34.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.971560 6682 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:03:34.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.972424 6682 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:34.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.972745 6682 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:34.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.995441 6682 rpc_server.cc:307] RPC server started. Bound to: 127.6.111.253:39259
08:03:34.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.995479 6758 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.6.111.253:39259 every 8 connection(s)
08:03:34.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:34.996958 6682 server_base.cc:1180] Dumped server information to /tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/master-1/data/info.pb
08:03:35.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.000545 6759 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:35.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.004072 6591 external_mini_cluster.cc:1413] Started /tmp/dist-test-task8V1MUJ/build/asan/bin/kudu as pid 6682
08:03:35.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.004244 6591 external_mini_cluster.cc:1427] Reading /tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/master-1/wal/instance
08:03:35.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.005540 6591 external_mini_cluster.cc:1351] Running /tmp/dist-test-task8V1MUJ/build/asan/bin/kudu
08:03:35.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8V1MUJ/build/asan/bin/kudu
08:03:35.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/master-2/wal
08:03:35.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/master-2/data
08:03:35.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:35.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:35.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:35.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:35.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:35.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:35.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/master-2/logs
08:03:35.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/master-2/data/info.pb
08:03:35.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:35.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:35.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:35.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:35.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:35.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:35.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:35.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:35.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:03:35.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:35.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:03:35.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:03:35.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.111.252:40087
08:03:35.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.111.252
08:03:35.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:35.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.111.212:42157
08:03:35.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:35.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:35.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:35.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:03:35.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.111.254:46321,127.6.111.253:39259,127.6.111.252:40087
08:03:35.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:03:35.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40499/jwks.json
08:03:35.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
08:03:35.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
08:03:35.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.007373 6759 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.111.254" port: 46321 } has no permanent_uuid. Determining permanent_uuid...
08:03:35.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.020947 6759 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.111.253" port: 39259 } has no permanent_uuid. Determining permanent_uuid...
08:03:35.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.030483 6759 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.111.252" port: 40087 } has no permanent_uuid. Determining permanent_uuid...
08:03:35.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:35.031464 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.111.252:40087: connect: Connection refused (error 111)
08:03:35.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:35.033066 6759 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.111.252:40087: Network error: Client connection negotiation failed: client connection to 127.6.111.252:40087: connect: Connection refused (error 111)
08:03:35.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.082419 6759 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.111.252" port: 40087 } attempt: 1
08:03:35.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:35.084596 6759 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.111.252:40087: Network error: Client connection negotiation failed: client connection to 127.6.111.252:40087: connect: Connection refused (error 111)
08:03:35.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.152892 6759 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.111.252" port: 40087 } attempt: 2
08:03:35.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:35.155026 6759 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.111.252:40087: Network error: Client connection negotiation failed: client connection to 127.6.111.252:40087: connect: Connection refused (error 111)
08:03:35.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.246312 6759 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.111.252" port: 40087 } attempt: 3
08:03:35.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:35.248273 6759 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.111.252:40087: Network error: Client connection negotiation failed: client connection to 127.6.111.252:40087: connect: Connection refused (error 111)
08:03:35.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.254738 6678 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.111.253" port: 39259 } attempt: 5
08:03:35.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.260761 6678 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.111.252" port: 40087 } has no permanent_uuid. Determining permanent_uuid...
08:03:35.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:35.262301 6678 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.111.252:40087: Network error: Client connection negotiation failed: client connection to 127.6.111.252:40087: connect: Connection refused (error 111)
08:03:35.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.313477 6678 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.111.252" port: 40087 } attempt: 1
08:03:35.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:35.315261 6678 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.111.252:40087: Network error: Client connection negotiation failed: client connection to 127.6.111.252:40087: connect: Connection refused (error 111)
08:03:35.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:35.319742 6761 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:03:35.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:35.320041 6761 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:35.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:35.320116 6761 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:03:35.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:35.327903 6761 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:03:35.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:35.327991 6761 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
08:03:35.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:35.328039 6761 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:03:35.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:35.328080 6761 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:03:35.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:35.328131 6761 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:03:35.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:35.328183 6761 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:03:35.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:35.328227 6761 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40499/jwks.json
08:03:35.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.337690 6761 master_runner.cc:386] Master server non-default flags:
08:03:35.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.111.212:42157
08:03:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
08:03:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/master-2/data
08:03:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/master-2/wal
08:03:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:03:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.111.254:46321,127.6.111.253:39259,127.6.111.252:40087
08:03:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
08:03:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:03:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.111.252:40087
08:03:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:03:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:03:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40499/jwks.json
08:03:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/master-2/data/info.pb
08:03:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.111.252
08:03:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/master-2/logs
08:03:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:03:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:03:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:03:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6d72d4a242076468501f3430b9a2cd050c634be2
08:03:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:03:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Jun 2025 07:43:22 UTC on 24a791456cd2
08:03:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6760
08:03:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
08:03:35.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.339227 6761 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.340978 6761 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:35.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:35.350817 6769 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:35.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:35.350862 6770 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:35.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:35.350929 6772 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:35.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.351954 6761 server_base.cc:1048] running on GCE node
08:03:35.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.352620 6761 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:35.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.353937 6761 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:35.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.355147 6761 hybrid_clock.cc:648] HybridClock initialized: now 1750838615355092 us; error 32 us; skew 500 ppm
08:03:35.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.355456 6761 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:03:35.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.357692 6761 webserver.cc:469] Webserver started at http://127.6.111.252:41967/ using document root <none> and password file <none>
08:03:35.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.358369 6761 fs_manager.cc:362] Metadata directory not provided
08:03:35.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.358484 6761 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:35.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.358852 6761 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
08:03:35.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.361164 6761 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/master-2/data/instance:
08:03:35.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "883f8f327b0e4b72b943de5ef25445f2"
08:03:35.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 08:03:35 on dist-test-slave-gp61"
08:03:35.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.361868 6761 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/master-2/wal/instance:
08:03:35.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "883f8f327b0e4b72b943de5ef25445f2"
08:03:35.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 08:03:35 on dist-test-slave-gp61"
08:03:35.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.366868 6761 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.005s sys 0.000s
08:03:35.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.369853 6778 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:03:35.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.371302 6761 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
08:03:35.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.371466 6761 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/master-2/data,/tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/master-2/wal
08:03:35.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "883f8f327b0e4b72b943de5ef25445f2"
08:03:35.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 08:03:35 on dist-test-slave-gp61"
08:03:35.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.371642 6761 fs_report.cc:389] FS layout report
08:03:35.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:35.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/master-2/wal
08:03:35.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/master-2/wal
08:03:35.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/master-2/data/data
08:03:35.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:35.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:35.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:35.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:35.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:35.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:35.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:35.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:35.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:35.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:35.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:35.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.383543 6678 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.111.252" port: 40087 } attempt: 2
08:03:35.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:35.385345 6678 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.111.252:40087: Network error: Client connection negotiation failed: client connection to 127.6.111.252:40087: connect: Connection refused (error 111)
08:03:35.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.387091 6761 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:03:35.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.387856 6761 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:35.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.388126 6761 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:35.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.391650 6759 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.111.252" port: 40087 } attempt: 4
08:03:35.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:35.393179 6759 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.111.252:40087: Network error: Client connection negotiation failed: client connection to 127.6.111.252:40087: connect: Connection refused (error 111)
08:03:35.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.410214 6761 rpc_server.cc:307] RPC server started. Bound to: 127.6.111.252:40087
08:03:35.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.410271 6839 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.6.111.252:40087 every 8 connection(s)
08:03:35.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.411617 6761 server_base.cc:1180] Dumped server information to /tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/master-2/data/info.pb
08:03:35.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.414208 6591 external_mini_cluster.cc:1413] Started /tmp/dist-test-task8V1MUJ/build/asan/bin/kudu as pid 6761
08:03:35.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.414355 6591 external_mini_cluster.cc:1427] Reading /tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/master-2/wal/instance
08:03:35.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.415377 6840 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:35.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.422770 6840 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.111.254" port: 46321 } has no permanent_uuid. Determining permanent_uuid...
08:03:35.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.435526 6840 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.111.253" port: 39259 } has no permanent_uuid. Determining permanent_uuid...
08:03:35.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.442071 6840 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.111.252" port: 40087 } has no permanent_uuid. Determining permanent_uuid...
08:03:35.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.451439 6840 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 883f8f327b0e4b72b943de5ef25445f2: Bootstrap starting.
08:03:35.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.454913 6840 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 883f8f327b0e4b72b943de5ef25445f2: Neither blocks nor log segments found. Creating new log.
08:03:35.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.455893 6840 log.cc:826] T 00000000000000000000000000000000 P 883f8f327b0e4b72b943de5ef25445f2: Log is configured to *not* fsync() on all Append() calls
08:03:35.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.458519 6840 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 883f8f327b0e4b72b943de5ef25445f2: No bootstrap required, opened a new log
08:03:35.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.463636 6840 raft_consensus.cc:357] T 00000000000000000000000000000000 P 883f8f327b0e4b72b943de5ef25445f2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "37df082dc4d74885bb14a018ac6c52a1" member_type: VOTER last_known_addr { host: "127.6.111.254" port: 46321 } } peers { permanent_uuid: "98a4571016524600ae1e653f157c03c0" member_type: VOTER last_known_addr { host: "127.6.111.253" port: 39259 } } peers { permanent_uuid: "883f8f327b0e4b72b943de5ef25445f2" member_type: VOTER last_known_addr { host: "127.6.111.252" port: 40087 } }
08:03:35.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.463902 6840 raft_consensus.cc:383] T 00000000000000000000000000000000 P 883f8f327b0e4b72b943de5ef25445f2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:35.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.464012 6840 raft_consensus.cc:738] T 00000000000000000000000000000000 P 883f8f327b0e4b72b943de5ef25445f2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 883f8f327b0e4b72b943de5ef25445f2, State: Initialized, Role: FOLLOWER
08:03:35.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.464809 6840 consensus_queue.cc:260] T 00000000000000000000000000000000 P 883f8f327b0e4b72b943de5ef25445f2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "37df082dc4d74885bb14a018ac6c52a1" member_type: VOTER last_known_addr { host: "127.6.111.254" port: 46321 } } peers { permanent_uuid: "98a4571016524600ae1e653f157c03c0" member_type: VOTER last_known_addr { host: "127.6.111.253" port: 39259 } } peers { permanent_uuid: "883f8f327b0e4b72b943de5ef25445f2" member_type: VOTER last_known_addr { host: "127.6.111.252" port: 40087 } }
08:03:35.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.465790 6847 sys_catalog.cc:455] T 00000000000000000000000000000000 P 883f8f327b0e4b72b943de5ef25445f2 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "37df082dc4d74885bb14a018ac6c52a1" member_type: VOTER last_known_addr { host: "127.6.111.254" port: 46321 } } peers { permanent_uuid: "98a4571016524600ae1e653f157c03c0" member_type: VOTER last_known_addr { host: "127.6.111.253" port: 39259 } } peers { permanent_uuid: "883f8f327b0e4b72b943de5ef25445f2" member_type: VOTER last_known_addr { host: "127.6.111.252" port: 40087 } } }
08:03:35.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.466087 6847 sys_catalog.cc:458] T 00000000000000000000000000000000 P 883f8f327b0e4b72b943de5ef25445f2 [sys.catalog]: This master's current role is: FOLLOWER
08:03:35.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.466698 6840 sys_catalog.cc:564] T 00000000000000000000000000000000 P 883f8f327b0e4b72b943de5ef25445f2 [sys.catalog]: configured and running, proceeding with master startup.
08:03:35.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:35.474946 6858 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 883f8f327b0e4b72b943de5ef25445f2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:03:35.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:35.475217 6858 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:03:35.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.472728 6761 master_runner.cc:186] Error getting master registration for 127.6.111.254:46321: OK, instance_id { permanent_uuid: "37df082dc4d74885bb14a018ac6c52a1" instance_seqno: 1750838614557253 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:03:35.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.479287 6761 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:03:35.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.491582 6678 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.111.252" port: 40087 } attempt: 3
08:03:35.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.501638 6678 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1: Bootstrap starting.
08:03:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.505339 6678 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1: Neither blocks nor log segments found. Creating new log.
08:03:35.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.506620 6678 log.cc:826] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1: Log is configured to *not* fsync() on all Append() calls
08:03:35.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.509469 6678 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1: No bootstrap required, opened a new log
08:03:35.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.516537 6678 raft_consensus.cc:357] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "37df082dc4d74885bb14a018ac6c52a1" member_type: VOTER last_known_addr { host: "127.6.111.254" port: 46321 } } peers { permanent_uuid: "98a4571016524600ae1e653f157c03c0" member_type: VOTER last_known_addr { host: "127.6.111.253" port: 39259 } } peers { permanent_uuid: "883f8f327b0e4b72b943de5ef25445f2" member_type: VOTER last_known_addr { host: "127.6.111.252" port: 40087 } }
08:03:35.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.516865 6678 raft_consensus.cc:383] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:35.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.516969 6678 raft_consensus.cc:738] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 37df082dc4d74885bb14a018ac6c52a1, State: Initialized, Role: FOLLOWER
08:03:35.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.517678 6678 consensus_queue.cc:260] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "37df082dc4d74885bb14a018ac6c52a1" member_type: VOTER last_known_addr { host: "127.6.111.254" port: 46321 } } peers { permanent_uuid: "98a4571016524600ae1e653f157c03c0" member_type: VOTER last_known_addr { host: "127.6.111.253" port: 39259 } } peers { permanent_uuid: "883f8f327b0e4b72b943de5ef25445f2" member_type: VOTER last_known_addr { host: "127.6.111.252" port: 40087 } }
08:03:35.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.518539 6861 sys_catalog.cc:455] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "37df082dc4d74885bb14a018ac6c52a1" member_type: VOTER last_known_addr { host: "127.6.111.254" port: 46321 } } peers { permanent_uuid: "98a4571016524600ae1e653f157c03c0" member_type: VOTER last_known_addr { host: "127.6.111.253" port: 39259 } } peers { permanent_uuid: "883f8f327b0e4b72b943de5ef25445f2" member_type: VOTER last_known_addr { host: "127.6.111.252" port: 40087 } } }
08:03:35.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.518954 6861 sys_catalog.cc:458] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [sys.catalog]: This master's current role is: FOLLOWER
08:03:35.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.520200 6678 sys_catalog.cc:564] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [sys.catalog]: configured and running, proceeding with master startup.
08:03:35.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:35.529973 6872 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:03:35.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:35.530088 6872 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:03:35.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.529044 6602 master_runner.cc:186] Error getting master registration for 127.6.111.253:39259: OK, instance_id { permanent_uuid: "98a4571016524600ae1e653f157c03c0" instance_seqno: 1750838614989505 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:03:35.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.534569 6602 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:03:35.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:35.542212 6622 tablet.cc:2378] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:03:35.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.552103 6847 raft_consensus.cc:491] T 00000000000000000000000000000000 P 883f8f327b0e4b72b943de5ef25445f2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:03:35.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.552347 6847 raft_consensus.cc:513] T 00000000000000000000000000000000 P 883f8f327b0e4b72b943de5ef25445f2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "37df082dc4d74885bb14a018ac6c52a1" member_type: VOTER last_known_addr { host: "127.6.111.254" port: 46321 } } peers { permanent_uuid: "98a4571016524600ae1e653f157c03c0" member_type: VOTER last_known_addr { host: "127.6.111.253" port: 39259 } } peers { permanent_uuid: "883f8f327b0e4b72b943de5ef25445f2" member_type: VOTER last_known_addr { host: "127.6.111.252" port: 40087 } }
08:03:35.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.553732 6847 leader_election.cc:290] T 00000000000000000000000000000000 P 883f8f327b0e4b72b943de5ef25445f2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 37df082dc4d74885bb14a018ac6c52a1 (127.6.111.254:46321), 98a4571016524600ae1e653f157c03c0 (127.6.111.253:39259)
08:03:35.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.554448 6643 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "883f8f327b0e4b72b943de5ef25445f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "37df082dc4d74885bb14a018ac6c52a1" is_pre_election: true
08:03:35.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.555001 6643 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 883f8f327b0e4b72b943de5ef25445f2 in term 0.
08:03:35.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.555598 6781 leader_election.cc:304] T 00000000000000000000000000000000 P 883f8f327b0e4b72b943de5ef25445f2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 37df082dc4d74885bb14a018ac6c52a1, 883f8f327b0e4b72b943de5ef25445f2; no voters:
08:03:35.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.555963 6847 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 883f8f327b0e4b72b943de5ef25445f2 [term 0 FOLLOWER]: Leader pre-election won for term 1
08:03:35.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.556093 6847 raft_consensus.cc:491] T 00000000000000000000000000000000 P 883f8f327b0e4b72b943de5ef25445f2 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:03:35.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.556202 6847 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 883f8f327b0e4b72b943de5ef25445f2 [term 0 FOLLOWER]: Advancing to term 1
08:03:35.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.559414 6847 raft_consensus.cc:513] T 00000000000000000000000000000000 P 883f8f327b0e4b72b943de5ef25445f2 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "37df082dc4d74885bb14a018ac6c52a1" member_type: VOTER last_known_addr { host: "127.6.111.254" port: 46321 } } peers { permanent_uuid: "98a4571016524600ae1e653f157c03c0" member_type: VOTER last_known_addr { host: "127.6.111.253" port: 39259 } } peers { permanent_uuid: "883f8f327b0e4b72b943de5ef25445f2" member_type: VOTER last_known_addr { host: "127.6.111.252" port: 40087 } }
08:03:35.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.560408 6847 leader_election.cc:290] T 00000000000000000000000000000000 P 883f8f327b0e4b72b943de5ef25445f2 [CANDIDATE]: Term 1 election: Requested vote from peers 37df082dc4d74885bb14a018ac6c52a1 (127.6.111.254:46321), 98a4571016524600ae1e653f157c03c0 (127.6.111.253:39259)
08:03:35.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.560441 6643 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "883f8f327b0e4b72b943de5ef25445f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "37df082dc4d74885bb14a018ac6c52a1"
08:03:35.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.560612 6643 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [term 0 FOLLOWER]: Advancing to term 1
08:03:35.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.563391 6643 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 883f8f327b0e4b72b943de5ef25445f2 in term 1.
08:03:35.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.563920 6781 leader_election.cc:304] T 00000000000000000000000000000000 P 883f8f327b0e4b72b943de5ef25445f2 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 37df082dc4d74885bb14a018ac6c52a1, 883f8f327b0e4b72b943de5ef25445f2; no voters:
08:03:35.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.564713 6847 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 883f8f327b0e4b72b943de5ef25445f2 [term 1 FOLLOWER]: Leader election won for term 1
08:03:35.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.586658 6847 raft_consensus.cc:695] T 00000000000000000000000000000000 P 883f8f327b0e4b72b943de5ef25445f2 [term 1 LEADER]: Becoming Leader. State: Replica: 883f8f327b0e4b72b943de5ef25445f2, State: Running, Role: LEADER
08:03:35.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.587541 6847 consensus_queue.cc:237] T 00000000000000000000000000000000 P 883f8f327b0e4b72b943de5ef25445f2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "37df082dc4d74885bb14a018ac6c52a1" member_type: VOTER last_known_addr { host: "127.6.111.254" port: 46321 } } peers { permanent_uuid: "98a4571016524600ae1e653f157c03c0" member_type: VOTER last_known_addr { host: "127.6.111.253" port: 39259 } } peers { permanent_uuid: "883f8f327b0e4b72b943de5ef25445f2" member_type: VOTER last_known_addr { host: "127.6.111.252" port: 40087 } }
08:03:35.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.599748 6874 sys_catalog.cc:455] T 00000000000000000000000000000000 P 883f8f327b0e4b72b943de5ef25445f2 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 883f8f327b0e4b72b943de5ef25445f2. Latest consensus state: current_term: 1 leader_uuid: "883f8f327b0e4b72b943de5ef25445f2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "37df082dc4d74885bb14a018ac6c52a1" member_type: VOTER last_known_addr { host: "127.6.111.254" port: 46321 } } peers { permanent_uuid: "98a4571016524600ae1e653f157c03c0" member_type: VOTER last_known_addr { host: "127.6.111.253" port: 39259 } } peers { permanent_uuid: "883f8f327b0e4b72b943de5ef25445f2" member_type: VOTER last_known_addr { host: "127.6.111.252" port: 40087 } } }
08:03:35.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.600260 6874 sys_catalog.cc:458] T 00000000000000000000000000000000 P 883f8f327b0e4b72b943de5ef25445f2 [sys.catalog]: This master's current role is: LEADER
08:03:35.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.607265 6876 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
08:03:35.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.609458 6876 catalog_manager.cc:1486] Initializing Kudu cluster ID...
08:03:35.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.622965 6643 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [term 1 FOLLOWER]: Refusing update from remote peer 883f8f327b0e4b72b943de5ef25445f2: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:03:35.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.624065 6874 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 883f8f327b0e4b72b943de5ef25445f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "37df082dc4d74885bb14a018ac6c52a1" member_type: VOTER last_known_addr { host: "127.6.111.254" port: 46321 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:03:35.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:35.640164 6784 tablet.cc:2378] T 00000000000000000000000000000000 P 883f8f327b0e4b72b943de5ef25445f2: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:03:35.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.648578 6861 sys_catalog.cc:455] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 883f8f327b0e4b72b943de5ef25445f2. Latest consensus state: current_term: 1 leader_uuid: "883f8f327b0e4b72b943de5ef25445f2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "37df082dc4d74885bb14a018ac6c52a1" member_type: VOTER last_known_addr { host: "127.6.111.254" port: 46321 } } peers { permanent_uuid: "98a4571016524600ae1e653f157c03c0" member_type: VOTER last_known_addr { host: "127.6.111.253" port: 39259 } } peers { permanent_uuid: "883f8f327b0e4b72b943de5ef25445f2" member_type: VOTER last_known_addr { host: "127.6.111.252" port: 40087 } } }
08:03:35.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.648880 6861 sys_catalog.cc:458] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [sys.catalog]: This master's current role is: FOLLOWER
08:03:35.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.652689 6874 sys_catalog.cc:455] T 00000000000000000000000000000000 P 883f8f327b0e4b72b943de5ef25445f2 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "883f8f327b0e4b72b943de5ef25445f2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "37df082dc4d74885bb14a018ac6c52a1" member_type: VOTER last_known_addr { host: "127.6.111.254" port: 46321 } } peers { permanent_uuid: "98a4571016524600ae1e653f157c03c0" member_type: VOTER last_known_addr { host: "127.6.111.253" port: 39259 } } peers { permanent_uuid: "883f8f327b0e4b72b943de5ef25445f2" member_type: VOTER last_known_addr { host: "127.6.111.252" port: 40087 } } }
08:03:35.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.652954 6874 sys_catalog.cc:458] T 00000000000000000000000000000000 P 883f8f327b0e4b72b943de5ef25445f2 [sys.catalog]: This master's current role is: LEADER
08:03:35.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.656810 6876 catalog_manager.cc:1349] Generated new cluster ID: f3216bb837ce46b28605338140db648e
08:03:35.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.656930 6876 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
08:03:35.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.665280 6861 sys_catalog.cc:455] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "883f8f327b0e4b72b943de5ef25445f2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "37df082dc4d74885bb14a018ac6c52a1" member_type: VOTER last_known_addr { host: "127.6.111.254" port: 46321 } } peers { permanent_uuid: "98a4571016524600ae1e653f157c03c0" member_type: VOTER last_known_addr { host: "127.6.111.253" port: 39259 } } peers { permanent_uuid: "883f8f327b0e4b72b943de5ef25445f2" member_type: VOTER last_known_addr { host: "127.6.111.252" port: 40087 } } }
08:03:35.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.665478 6861 sys_catalog.cc:458] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [sys.catalog]: This master's current role is: FOLLOWER
08:03:35.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.680541 6876 catalog_manager.cc:1372] Generated new certificate authority record
08:03:35.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.682322 6876 catalog_manager.cc:1506] Loading token signing keys...
08:03:35.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.692448 6759 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.111.252" port: 40087 } attempt: 5
08:03:35.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.706014 6759 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0: Bootstrap starting.
08:03:35.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.710216 6759 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0: Neither blocks nor log segments found. Creating new log.
08:03:35.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.711570 6759 log.cc:826] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0: Log is configured to *not* fsync() on all Append() calls
08:03:35.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.711906 6876 catalog_manager.cc:5955] T 00000000000000000000000000000000 P 883f8f327b0e4b72b943de5ef25445f2: Generated new TSK 0
08:03:35.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.712961 6876 catalog_manager.cc:1516] Initializing in-progress tserver states...
08:03:35.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.717535 6759 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0: No bootstrap required, opened a new log
08:03:35.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.725318 6759 raft_consensus.cc:357] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "37df082dc4d74885bb14a018ac6c52a1" member_type: VOTER last_known_addr { host: "127.6.111.254" port: 46321 } } peers { permanent_uuid: "98a4571016524600ae1e653f157c03c0" member_type: VOTER last_known_addr { host: "127.6.111.253" port: 39259 } } peers { permanent_uuid: "883f8f327b0e4b72b943de5ef25445f2" member_type: VOTER last_known_addr { host: "127.6.111.252" port: 40087 } }
08:03:35.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.725747 6759 raft_consensus.cc:383] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:35.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.725872 6759 raft_consensus.cc:738] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 98a4571016524600ae1e653f157c03c0, State: Initialized, Role: FOLLOWER
08:03:35.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.726676 6759 consensus_queue.cc:260] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "37df082dc4d74885bb14a018ac6c52a1" member_type: VOTER last_known_addr { host: "127.6.111.254" port: 46321 } } peers { permanent_uuid: "98a4571016524600ae1e653f157c03c0" member_type: VOTER last_known_addr { host: "127.6.111.253" port: 39259 } } peers { permanent_uuid: "883f8f327b0e4b72b943de5ef25445f2" member_type: VOTER last_known_addr { host: "127.6.111.252" port: 40087 } }
08:03:35.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.728332 6883 sys_catalog.cc:455] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "37df082dc4d74885bb14a018ac6c52a1" member_type: VOTER last_known_addr { host: "127.6.111.254" port: 46321 } } peers { permanent_uuid: "98a4571016524600ae1e653f157c03c0" member_type: VOTER last_known_addr { host: "127.6.111.253" port: 39259 } } peers { permanent_uuid: "883f8f327b0e4b72b943de5ef25445f2" member_type: VOTER last_known_addr { host: "127.6.111.252" port: 40087 } } }
08:03:35.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.729240 6759 sys_catalog.cc:564] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0 [sys.catalog]: configured and running, proceeding with master startup.
08:03:35.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.729281 6723 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "883f8f327b0e4b72b943de5ef25445f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "98a4571016524600ae1e653f157c03c0"
08:03:35.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.729749 6723 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0 [term 0 FOLLOWER]: Advancing to term 1
08:03:35.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.732748 6724 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "883f8f327b0e4b72b943de5ef25445f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "98a4571016524600ae1e653f157c03c0" is_pre_election: true
08:03:35.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.741508 6723 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 883f8f327b0e4b72b943de5ef25445f2 in term 1.
08:03:35.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.729076 6883 sys_catalog.cc:458] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0 [sys.catalog]: This master's current role is: FOLLOWER
08:03:35.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.742213 6722 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0 [term 1 FOLLOWER]: Refusing update from remote peer 883f8f327b0e4b72b943de5ef25445f2: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:03:35.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.743224 6591 external_mini_cluster.cc:1351] Running /tmp/dist-test-task8V1MUJ/build/asan/bin/kudu
08:03:35.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8V1MUJ/build/asan/bin/kudu
08:03:35.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/ts-0/wal
08:03:35.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/ts-0/data
08:03:35.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:35.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:35.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:35.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:35.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:35.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:35.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/ts-0/logs
08:03:35.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/ts-0/data/info.pb
08:03:35.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:35.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:35.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:35.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:35.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:35.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:35.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:35.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:35.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:03:35.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:35.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.111.193:0
08:03:35.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.111.193
08:03:35.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.111.193
08:03:35.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:35.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.111.254:46321,127.6.111.253:39259,127.6.111.252:40087
08:03:35.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.111.212:42157
08:03:35.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:35.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:35.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:35.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
08:03:35.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
08:03:35.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
08:03:35.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.746603 6874 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 883f8f327b0e4b72b943de5ef25445f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "98a4571016524600ae1e653f157c03c0" member_type: VOTER last_known_addr { host: "127.6.111.253" port: 39259 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:03:35.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.779773 6887 mvcc.cc:204] Tried to move back new op lower bound from 7171434969580040192 to 7171434969451675648. Current Snapshot: MvccSnapshot[applied={T|T < 7171434969580040192}]
08:03:35.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.794446 6883 sys_catalog.cc:455] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 883f8f327b0e4b72b943de5ef25445f2. Latest consensus state: current_term: 1 leader_uuid: "883f8f327b0e4b72b943de5ef25445f2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "37df082dc4d74885bb14a018ac6c52a1" member_type: VOTER last_known_addr { host: "127.6.111.254" port: 46321 } } peers { permanent_uuid: "98a4571016524600ae1e653f157c03c0" member_type: VOTER last_known_addr { host: "127.6.111.253" port: 39259 } } peers { permanent_uuid: "883f8f327b0e4b72b943de5ef25445f2" member_type: VOTER last_known_addr { host: "127.6.111.252" port: 40087 } } }
08:03:35.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.794430 6890 sys_catalog.cc:455] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "883f8f327b0e4b72b943de5ef25445f2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "37df082dc4d74885bb14a018ac6c52a1" member_type: VOTER last_known_addr { host: "127.6.111.254" port: 46321 } } peers { permanent_uuid: "98a4571016524600ae1e653f157c03c0" member_type: VOTER last_known_addr { host: "127.6.111.253" port: 39259 } } peers { permanent_uuid: "883f8f327b0e4b72b943de5ef25445f2" member_type: VOTER last_known_addr { host: "127.6.111.252" port: 40087 } } }
08:03:35.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.794760 6883 sys_catalog.cc:458] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0 [sys.catalog]: This master's current role is: FOLLOWER
08:03:35.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.794791 6890 sys_catalog.cc:458] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0 [sys.catalog]: This master's current role is: FOLLOWER
08:03:35.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.845034 6900 catalog_manager.cc:797] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
08:03:35.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.857795 6874 sys_catalog.cc:455] T 00000000000000000000000000000000 P 883f8f327b0e4b72b943de5ef25445f2 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "883f8f327b0e4b72b943de5ef25445f2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "37df082dc4d74885bb14a018ac6c52a1" member_type: VOTER last_known_addr { host: "127.6.111.254" port: 46321 } } peers { permanent_uuid: "98a4571016524600ae1e653f157c03c0" member_type: VOTER last_known_addr { host: "127.6.111.253" port: 39259 } } peers { permanent_uuid: "883f8f327b0e4b72b943de5ef25445f2" member_type: VOTER last_known_addr { host: "127.6.111.252" port: 40087 } } }
08:03:35.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:35.858124 6874 sys_catalog.cc:458] T 00000000000000000000000000000000 P 883f8f327b0e4b72b943de5ef25445f2 [sys.catalog]: This master's current role is: LEADER
08:03:36.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:36.150492 6886 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:03:36.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:36.150791 6886 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:36.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:36.150885 6886 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:03:36.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:36.158560 6886 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:03:36.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:36.158721 6886 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
08:03:36.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:36.158775 6886 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
08:03:36.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:36.158812 6886 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
08:03:36.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:36.158892 6886 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.111.193
08:03:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.168327 6886 tablet_server_runner.cc:78] Tablet server non-default flags:
08:03:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.111.212:42157
08:03:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/ts-0/data
08:03:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/ts-0/wal
08:03:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.111.193:0
08:03:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/ts-0/data/info.pb
08:03:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.111.193
08:03:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
08:03:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
08:03:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
08:03:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.111.254:46321,127.6.111.253:39259,127.6.111.252:40087
08:03:36.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:36.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:36.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:36.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:36.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:36.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.111.193
08:03:36.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/ts-0/logs
08:03:36.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:36.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:36.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:03:36.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:03:36.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:03:36.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6d72d4a242076468501f3430b9a2cd050c634be2
08:03:36.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:03:36.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Jun 2025 07:43:22 UTC on 24a791456cd2
08:03:36.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6760
08:03:36.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
08:03:36.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.170010 6886 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:36.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.171788 6886 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:36.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:36.182926 6906 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:36.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:36.183176 6909 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:36.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:36.182983 6907 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:36.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.184118 6886 server_base.cc:1048] running on GCE node
08:03:36.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.184635 6886 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:36.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.185539 6886 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:36.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.186754 6886 hybrid_clock.cc:648] HybridClock initialized: now 1750838616186687 us; error 45 us; skew 500 ppm
08:03:36.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.187039 6886 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:03:36.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.189309 6886 webserver.cc:469] Webserver started at http://127.6.111.193:44717/ using document root <none> and password file <none>
08:03:36.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.190001 6886 fs_manager.cc:362] Metadata directory not provided
08:03:36.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.190125 6886 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:36.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.190547 6886 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
08:03:36.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.193245 6886 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/ts-0/data/instance:
08:03:36.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f7b24a2ea005466d8b25624cc1a9044f"
08:03:36.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 08:03:36 on dist-test-slave-gp61"
08:03:36.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.194015 6886 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/ts-0/wal/instance:
08:03:36.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f7b24a2ea005466d8b25624cc1a9044f"
08:03:36.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 08:03:36 on dist-test-slave-gp61"
08:03:36.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.199375 6886 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.006s sys 0.000s
08:03:36.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.202692 6915 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:03:36.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.203980 6886 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
08:03:36.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.204110 6886 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/ts-0/data,/tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/ts-0/wal
08:03:36.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f7b24a2ea005466d8b25624cc1a9044f"
08:03:36.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 08:03:36 on dist-test-slave-gp61"
08:03:36.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.204260 6886 fs_report.cc:389] FS layout report
08:03:36.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:36.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/ts-0/wal
08:03:36.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/ts-0/wal
08:03:36.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/ts-0/data/data
08:03:36.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:36.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:36.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:36.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:36.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:36.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:36.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:36.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:36.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:36.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:36.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:36.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.224961 6886 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:03:36.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.225771 6886 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:36.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.226091 6886 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:36.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.229712 6886 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:03:36.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.229990 6886 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.001s
08:03:36.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.230242 6886 ts_tablet_manager.cc:610] Registered 0 tablets
08:03:36.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.230335 6886 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:03:36.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.283784 6886 rpc_server.cc:307] RPC server started. Bound to: 127.6.111.193:36349
08:03:36.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.283846 7039 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.6.111.193:36349 every 8 connection(s)
08:03:36.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.285106 6886 server_base.cc:1180] Dumped server information to /tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/ts-0/data/info.pb
08:03:36.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.288529 7041 heartbeater.cc:344] Connected to a master server at 127.6.111.254:46321
08:03:36.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.289563 7041 heartbeater.cc:461] Registering TS with master...
08:03:36.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.289629 6591 external_mini_cluster.cc:1413] Started /tmp/dist-test-task8V1MUJ/build/asan/bin/kudu as pid 6886
08:03:36.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.289808 6591 external_mini_cluster.cc:1427] Reading /tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/ts-0/wal/instance
08:03:36.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.290627 7041 heartbeater.cc:507] Master 127.6.111.254:46321 requested a full tablet report, sending...
08:03:36.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.293764 6633 ts_manager.cc:194] Registered new tserver with Master: f7b24a2ea005466d8b25624cc1a9044f (127.6.111.193:36349)
08:03:36.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.295719 7040 heartbeater.cc:344] Connected to a master server at 127.6.111.252:40087
08:03:36.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.295816 6591 external_mini_cluster.cc:1351] Running /tmp/dist-test-task8V1MUJ/build/asan/bin/kudu
08:03:36.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8V1MUJ/build/asan/bin/kudu
08:03:36.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/ts-1/wal
08:03:36.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/ts-1/data
08:03:36.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:36.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:36.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:36.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:36.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:36.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:36.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/ts-1/logs
08:03:36.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/ts-1/data/info.pb
08:03:36.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:36.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:36.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:36.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:36.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:36.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:36.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:36.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:36.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:03:36.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:36.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.111.194:0
08:03:36.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.111.194
08:03:36.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.111.194
08:03:36.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:36.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.111.254:46321,127.6.111.253:39259,127.6.111.252:40087
08:03:36.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.111.212:42157
08:03:36.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:36.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:36.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:36.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
08:03:36.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
08:03:36.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
08:03:36.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.295871 7040 heartbeater.cc:461] Registering TS with master...
08:03:36.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.296419 7040 heartbeater.cc:507] Master 127.6.111.252:40087 requested a full tablet report, sending...
08:03:36.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.298225 6795 ts_manager.cc:194] Registered new tserver with Master: f7b24a2ea005466d8b25624cc1a9044f (127.6.111.193:36349)
08:03:36.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.299603 7042 heartbeater.cc:344] Connected to a master server at 127.6.111.253:39259
08:03:36.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.299734 7042 heartbeater.cc:461] Registering TS with master...
08:03:36.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.300180 7042 heartbeater.cc:507] Master 127.6.111.253:39259 requested a full tablet report, sending...
08:03:36.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.300155 6795 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.6.111.193:60163
08:03:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.301647 6714 ts_manager.cc:194] Registered new tserver with Master: f7b24a2ea005466d8b25624cc1a9044f (127.6.111.193:36349)
08:03:36.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.532023 6872 catalog_manager.cc:1261] Loaded cluster ID: f3216bb837ce46b28605338140db648e
08:03:36.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.532121 6872 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1: loading cluster ID for follower catalog manager: success
08:03:36.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.534634 6872 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1: acquiring CA information for follower catalog manager: success
08:03:36.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.536437 6872 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:03:36.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:36.606933 7045 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:03:36.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:36.607203 7045 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:36.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:36.607295 7045 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:03:36.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:36.614948 7045 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:03:36.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:36.615070 7045 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
08:03:36.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:36.615115 7045 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
08:03:36.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:36.615146 7045 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
08:03:36.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:36.615235 7045 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.111.194
08:03:36.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.623827 7045 tablet_server_runner.cc:78] Tablet server non-default flags:
08:03:36.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:36.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.111.212:42157
08:03:36.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:36.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:36.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/ts-1/data
08:03:36.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/ts-1/wal
08:03:36.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:36.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:36.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.111.194:0
08:03:36.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:36.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:36.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:36.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/ts-1/data/info.pb
08:03:36.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.111.194
08:03:36.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:36.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
08:03:36.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
08:03:36.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
08:03:36.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.111.254:46321,127.6.111.253:39259,127.6.111.252:40087
08:03:36.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:36.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:36.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:36.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:36.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:36.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.111.194
08:03:36.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/ts-1/logs
08:03:36.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:36.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:36.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:03:36.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:03:36.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:03:36.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6d72d4a242076468501f3430b9a2cd050c634be2
08:03:36.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:03:36.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Jun 2025 07:43:22 UTC on 24a791456cd2
08:03:36.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6760
08:03:36.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
08:03:36.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.625355 7045 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:36.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.627012 7045 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:36.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:36.638125 7053 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:36.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:36.638095 7051 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:36.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:36.638051 7050 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:36.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.639192 7045 server_base.cc:1048] running on GCE node
08:03:36.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.639781 7045 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:36.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.640574 7045 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:36.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.641789 7045 hybrid_clock.cc:648] HybridClock initialized: now 1750838616641721 us; error 41 us; skew 500 ppm
08:03:36.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.642067 7045 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:03:36.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.644286 7045 webserver.cc:469] Webserver started at http://127.6.111.194:33365/ using document root <none> and password file <none>
08:03:36.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.645115 7045 fs_manager.cc:362] Metadata directory not provided
08:03:36.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.645253 7045 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:36.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.645653 7045 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
08:03:36.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.648175 7045 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/ts-1/data/instance:
08:03:36.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "090753c2c8b748e9a1d89d70c8001722"
08:03:36.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 08:03:36 on dist-test-slave-gp61"
08:03:36.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.649077 7045 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/ts-1/wal/instance:
08:03:36.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "090753c2c8b748e9a1d89d70c8001722"
08:03:36.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 08:03:36 on dist-test-slave-gp61"
08:03:36.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.654036 7045 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.002s sys 0.002s
08:03:36.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.657177 7059 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:03:36.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.658366 7045 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.002s
08:03:36.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.658517 7045 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/ts-1/data,/tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/ts-1/wal
08:03:36.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "090753c2c8b748e9a1d89d70c8001722"
08:03:36.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 08:03:36 on dist-test-slave-gp61"
08:03:36.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.658740 7045 fs_report.cc:389] FS layout report
08:03:36.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:36.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/ts-1/wal
08:03:36.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/ts-1/wal
08:03:36.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/ts-1/data/data
08:03:36.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:36.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:36.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:36.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:36.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:36.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:36.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:36.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:36.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:36.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:36.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:36.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.686333 7045 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:03:36.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.687101 7045 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:36.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.687427 7045 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:36.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.690693 7045 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:03:36.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.690822 7045 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
08:03:36.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.690958 7045 ts_tablet_manager.cc:610] Registered 0 tablets
08:03:36.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.691051 7045 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:03:36.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.749491 7045 rpc_server.cc:307] RPC server started. Bound to: 127.6.111.194:44167
08:03:36.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.749526 7180 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.6.111.194:44167 every 8 connection(s)
08:03:36.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.750802 7045 server_base.cc:1180] Dumped server information to /tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/ts-1/data/info.pb
08:03:36.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.755028 6591 external_mini_cluster.cc:1413] Started /tmp/dist-test-task8V1MUJ/build/asan/bin/kudu as pid 7045
08:03:36.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.755149 7182 heartbeater.cc:344] Connected to a master server at 127.6.111.254:46321
08:03:36.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.755534 7182 heartbeater.cc:461] Registering TS with master...
08:03:36.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.755609 6591 external_mini_cluster.cc:1427] Reading /tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/ts-1/wal/instance
08:03:36.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.756987 7182 heartbeater.cc:507] Master 127.6.111.254:46321 requested a full tablet report, sending...
08:03:36.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.759943 6632 ts_manager.cc:194] Registered new tserver with Master: 090753c2c8b748e9a1d89d70c8001722 (127.6.111.194:44167)
08:03:36.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.761399 6591 external_mini_cluster.cc:1351] Running /tmp/dist-test-task8V1MUJ/build/asan/bin/kudu
08:03:36.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8V1MUJ/build/asan/bin/kudu
08:03:36.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/ts-2/wal
08:03:36.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/ts-2/data
08:03:36.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:36.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:36.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:36.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:36.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:36.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:36.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/ts-2/logs
08:03:36.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/ts-2/data/info.pb
08:03:36.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:36.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:36.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:36.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:36.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:36.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:36.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:36.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:36.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:03:36.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:36.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.111.195:0
08:03:36.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.111.195
08:03:36.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.111.195
08:03:36.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:36.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.111.254:46321,127.6.111.253:39259,127.6.111.252:40087
08:03:36.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.111.212:42157
08:03:36.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:36.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:36.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:36.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
08:03:36.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
08:03:36.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
08:03:36.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.766461 7181 heartbeater.cc:344] Connected to a master server at 127.6.111.252:40087
08:03:36.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.766649 7181 heartbeater.cc:461] Registering TS with master...
08:03:36.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.767110 7181 heartbeater.cc:507] Master 127.6.111.252:40087 requested a full tablet report, sending...
08:03:36.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.768416 6795 ts_manager.cc:194] Registered new tserver with Master: 090753c2c8b748e9a1d89d70c8001722 (127.6.111.194:44167)
08:03:36.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.769287 6795 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.6.111.194:39709
08:03:36.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.772451 7183 heartbeater.cc:344] Connected to a master server at 127.6.111.253:39259
08:03:36.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.772608 7183 heartbeater.cc:461] Registering TS with master...
08:03:36.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.772962 7183 heartbeater.cc:507] Master 127.6.111.253:39259 requested a full tablet report, sending...
08:03:36.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.773847 6714 ts_manager.cc:194] Registered new tserver with Master: 090753c2c8b748e9a1d89d70c8001722 (127.6.111.194:44167)
08:03:36.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.849855 6900 catalog_manager.cc:1261] Loaded cluster ID: f3216bb837ce46b28605338140db648e
08:03:36.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.849984 6900 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0: loading cluster ID for follower catalog manager: success
08:03:36.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.853441 6900 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0: acquiring CA information for follower catalog manager: success
08:03:36.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:36.855911 6900 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:03:37.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:37.073141 7186 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:03:37.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:37.073449 7186 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:37.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:37.073594 7186 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:03:37.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:37.081547 7186 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:03:37.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:37.081715 7186 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
08:03:37.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:37.081796 7186 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
08:03:37.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:37.081882 7186 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
08:03:37.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:37.082019 7186 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.111.195
08:03:37.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.091434 7186 tablet_server_runner.cc:78] Tablet server non-default flags:
08:03:37.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:37.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.111.212:42157
08:03:37.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:37.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:37.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/ts-2/data
08:03:37.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/ts-2/wal
08:03:37.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:37.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:37.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.111.195:0
08:03:37.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:37.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:37.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:37.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/ts-2/data/info.pb
08:03:37.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.111.195
08:03:37.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:37.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
08:03:37.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
08:03:37.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
08:03:37.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.111.254:46321,127.6.111.253:39259,127.6.111.252:40087
08:03:37.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:37.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:37.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:37.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:37.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:37.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.111.195
08:03:37.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/ts-2/logs
08:03:37.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:37.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:37.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:03:37.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:03:37.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:03:37.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6d72d4a242076468501f3430b9a2cd050c634be2
08:03:37.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:03:37.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Jun 2025 07:43:22 UTC on 24a791456cd2
08:03:37.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6760
08:03:37.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
08:03:37.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.093236 7186 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:37.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.095134 7186 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:37.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:37.107642 7193 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:37.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:37.108018 7195 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:37.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:37.108482 7192 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:37.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.109666 7186 server_base.cc:1048] running on GCE node
08:03:37.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.110353 7186 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:37.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.111539 7186 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:37.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.112828 7186 hybrid_clock.cc:648] HybridClock initialized: now 1750838617112751 us; error 43 us; skew 500 ppm
08:03:37.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.113229 7186 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:03:37.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.115743 7186 webserver.cc:469] Webserver started at http://127.6.111.195:33483/ using document root <none> and password file <none>
08:03:37.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.116432 7186 fs_manager.cc:362] Metadata directory not provided
08:03:37.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.116559 7186 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:37.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.116984 7186 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
08:03:37.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.119298 7186 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/ts-2/data/instance:
08:03:37.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ed40a832914747859ad22b79871417e6"
08:03:37.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 08:03:37 on dist-test-slave-gp61"
08:03:37.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.120096 7186 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/ts-2/wal/instance:
08:03:37.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ed40a832914747859ad22b79871417e6"
08:03:37.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 08:03:37 on dist-test-slave-gp61"
08:03:37.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.125643 7186 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.007s sys 0.000s
08:03:37.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.128924 7201 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:03:37.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.130205 7186 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.001s
08:03:37.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.130427 7186 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/ts-2/data,/tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/ts-2/wal
08:03:37.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ed40a832914747859ad22b79871417e6"
08:03:37.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 08:03:37 on dist-test-slave-gp61"
08:03:37.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.130584 7186 fs_report.cc:389] FS layout report
08:03:37.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:37.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/ts-2/wal
08:03:37.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/ts-2/wal
08:03:37.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/ts-2/data/data
08:03:37.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:37.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:37.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:37.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:37.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:37.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:37.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:37.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:37.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:37.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:37.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:37.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.166718 7186 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:03:37.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.167604 7186 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:37.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.167897 7186 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:37.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.171281 7186 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:03:37.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.171401 7186 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
08:03:37.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.171512 7186 ts_tablet_manager.cc:610] Registered 0 tablets
08:03:37.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.171626 7186 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:03:37.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.250277 7186 rpc_server.cc:307] RPC server started. Bound to: 127.6.111.195:33209
08:03:37.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.250322 7322 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.6.111.195:33209 every 8 connection(s)
08:03:37.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.251550 7186 server_base.cc:1180] Dumped server information to /tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/ts-2/data/info.pb
08:03:37.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.254233 7324 heartbeater.cc:344] Connected to a master server at 127.6.111.254:46321
08:03:37.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.254756 7324 heartbeater.cc:461] Registering TS with master...
08:03:37.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.256001 7324 heartbeater.cc:507] Master 127.6.111.254:46321 requested a full tablet report, sending...
08:03:37.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.258358 6632 ts_manager.cc:194] Registered new tserver with Master: ed40a832914747859ad22b79871417e6 (127.6.111.195:33209)
08:03:37.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.261267 6591 external_mini_cluster.cc:1413] Started /tmp/dist-test-task8V1MUJ/build/asan/bin/kudu as pid 7186
08:03:37.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.261440 6591 external_mini_cluster.cc:1427] Reading /tmp/dist-test-task8V1MUJ/test-tmp/mini-kudu-cluster9166308745084712022/ts-2/wal/instance
08:03:37.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.262759 7323 heartbeater.cc:344] Connected to a master server at 127.6.111.252:40087
08:03:37.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.262908 7323 heartbeater.cc:461] Registering TS with master...
08:03:37.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.263435 7323 heartbeater.cc:507] Master 127.6.111.252:40087 requested a full tablet report, sending...
08:03:37.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.264302 7325 heartbeater.cc:344] Connected to a master server at 127.6.111.253:39259
08:03:37.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.264457 6795 ts_manager.cc:194] Registered new tserver with Master: ed40a832914747859ad22b79871417e6 (127.6.111.195:33209)
08:03:37.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.264952 7325 heartbeater.cc:461] Registering TS with master...
08:03:37.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.265352 7325 heartbeater.cc:507] Master 127.6.111.253:39259 requested a full tablet report, sending...
08:03:37.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.265604 6795 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.6.111.195:35523
08:03:37.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.266417 6714 ts_manager.cc:194] Registered new tserver with Master: ed40a832914747859ad22b79871417e6 (127.6.111.195:33209)
08:03:37.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.270489 6591 external_mini_cluster.cc:934] 3 TS(s) registered with all masters
08:03:37.272 [DEBUG - main] (MiniKuduCluster.java:195) Response:
08:03:37.272 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
08:03:37.273 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.6.111.254"
port: 46321
}
bound_http_address {
host: "127.6.111.254"
port: 39873
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.6.111.253"
port: 39259
}
bound_http_address {
host: "127.6.111.253"
port: 32801
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.6.111.252"
port: 40087
}
bound_http_address {
host: "127.6.111.252"
port: 41967
}
}
}
08:03:37.273 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
08:03:37.274 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.6.111.193"
port: 36349
}
bound_http_address {
host: "127.6.111.193"
port: 44717
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.6.111.194"
port: 44167
}
bound_http_address {
host: "127.6.111.194"
port: 33365
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.6.111.195"
port: 33209
}
bound_http_address {
host: "127.6.111.195"
port: 33483
}
}
}
08:03:37.274 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
08:03:37.275 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
08:03:37.277 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.6.111.252:40087(127.6.111.252:40087)] connecting to peer
08:03:37.278 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.6.111.253:39259(127.6.111.253:39259)] connecting to peer
08:03:37.278 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.6.111.254:46321(127.6.111.254:46321)] connecting to peer
08:03:37.279 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.6.111.252:40087(127.6.111.252:40087)] Successfully connected to peer
08:03:37.280 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.6.111.254:46321(127.6.111.254:46321)] Successfully connected to peer
08:03:37.280 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.6.111.253:39259(127.6.111.253:39259)] Successfully connected to peer
08:03:37.281 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.6.111.252
08:03:37.282 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.6.111.253
08:03:37.293 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.6.111.254
08:03:37.294 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x2dfe5863, L:/127.0.0.1:60710 - R:/127.6.111.253:39259] using SASL/PLAIN
08:03:37.299 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x3c88e233, L:/127.0.0.1:50542 - R:/127.6.111.252:40087] using SASL/PLAIN
08:03:37.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.302932 7040 heartbeater.cc:499] Master 127.6.111.252:40087 was elected leader, sending a full tablet report...
08:03:37.306 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
08:03:37.306 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=29), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.252:40087, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.253:39259, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.254:46321, [20ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.253:39259: OK, [29ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.252:40087: OK, deferred=null) after lookup
08:03:37.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.307782 6795 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:50542:
08:03:37.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_manager_fallback"
08:03:37.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
08:03:37.308 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xd090e22a, L:/127.0.0.1:51186 - R:/127.6.111.254:46321] using SASL/PLAIN
08:03:37.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:37.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
08:03:37.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
08:03:37.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:03:37.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:37.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:37.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:37.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
08:03:37.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:37.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:37.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column1_i"
08:03:37.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
08:03:37.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
08:03:37.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:37.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:37.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:37.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
08:03:37.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:37.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:37.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column2_i"
08:03:37.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
08:03:37.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
08:03:37.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
08:03:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column3_s"
08:03:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
08:03:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
08:03:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
08:03:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: DICT_ENCODING
08:03:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: LZ4
08:03:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 4096
08:03:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
08:03:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column4_b"
08:03:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: BOOL
08:03:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
08:03:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
08:03:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
08:03:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) hash_schema {
08:03:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
08:03:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_buckets: 2
08:03:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) seed: 0
08:03:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:37.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:37.310154 6795 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table txn_manager_fallback in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
08:03:37.327 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table e46ea1d2e29e4e89869fdbc2f3d47cdc
08:03:37.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.373816 6975 tablet_service.cc:1468] Processing CreateTablet for tablet 7133300aa73e47f6aae2dd4cc4b7a030 (DEFAULT_TABLE table=txn_manager_fallback [id=e46ea1d2e29e4e89869fdbc2f3d47cdc]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
08:03:37.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.374305 7252 tablet_service.cc:1468] Processing CreateTablet for tablet 7133300aa73e47f6aae2dd4cc4b7a030 (DEFAULT_TABLE table=txn_manager_fallback [id=e46ea1d2e29e4e89869fdbc2f3d47cdc]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
08:03:37.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.374744 7115 tablet_service.cc:1468] Processing CreateTablet for tablet 469e5c7686c0457dab3668241a8bd9e0 (DEFAULT_TABLE table=txn_manager_fallback [id=e46ea1d2e29e4e89869fdbc2f3d47cdc]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
08:03:37.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.375602 6975 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 7133300aa73e47f6aae2dd4cc4b7a030. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:37.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.375627 7115 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 469e5c7686c0457dab3668241a8bd9e0. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:37.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.375574 7251 tablet_service.cc:1468] Processing CreateTablet for tablet 469e5c7686c0457dab3668241a8bd9e0 (DEFAULT_TABLE table=txn_manager_fallback [id=e46ea1d2e29e4e89869fdbc2f3d47cdc]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
08:03:37.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.376039 7252 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 7133300aa73e47f6aae2dd4cc4b7a030. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:37.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.373800 6973 tablet_service.cc:1468] Processing CreateTablet for tablet 469e5c7686c0457dab3668241a8bd9e0 (DEFAULT_TABLE table=txn_manager_fallback [id=e46ea1d2e29e4e89869fdbc2f3d47cdc]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
08:03:37.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.376648 6973 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 469e5c7686c0457dab3668241a8bd9e0. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:37.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.377408 7251 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 469e5c7686c0457dab3668241a8bd9e0. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:37.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.374305 7116 tablet_service.cc:1468] Processing CreateTablet for tablet 7133300aa73e47f6aae2dd4cc4b7a030 (DEFAULT_TABLE table=txn_manager_fallback [id=e46ea1d2e29e4e89869fdbc2f3d47cdc]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
08:03:37.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.378464 7116 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 7133300aa73e47f6aae2dd4cc4b7a030. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:37.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.398885 7349 tablet_bootstrap.cc:492] T 469e5c7686c0457dab3668241a8bd9e0 P ed40a832914747859ad22b79871417e6: Bootstrap starting.
08:03:37.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.400585 7350 tablet_bootstrap.cc:492] T 469e5c7686c0457dab3668241a8bd9e0 P 090753c2c8b748e9a1d89d70c8001722: Bootstrap starting.
08:03:37.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.402310 7349 tablet_bootstrap.cc:654] T 469e5c7686c0457dab3668241a8bd9e0 P ed40a832914747859ad22b79871417e6: Neither blocks nor log segments found. Creating new log.
08:03:37.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.402824 7350 tablet_bootstrap.cc:654] T 469e5c7686c0457dab3668241a8bd9e0 P 090753c2c8b748e9a1d89d70c8001722: Neither blocks nor log segments found. Creating new log.
08:03:37.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.403443 7349 log.cc:826] T 469e5c7686c0457dab3668241a8bd9e0 P ed40a832914747859ad22b79871417e6: Log is configured to *not* fsync() on all Append() calls
08:03:37.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.403705 7350 log.cc:826] T 469e5c7686c0457dab3668241a8bd9e0 P 090753c2c8b748e9a1d89d70c8001722: Log is configured to *not* fsync() on all Append() calls
08:03:37.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.404536 7348 tablet_bootstrap.cc:492] T 469e5c7686c0457dab3668241a8bd9e0 P f7b24a2ea005466d8b25624cc1a9044f: Bootstrap starting.
08:03:37.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.406219 7349 tablet_bootstrap.cc:492] T 469e5c7686c0457dab3668241a8bd9e0 P ed40a832914747859ad22b79871417e6: No bootstrap required, opened a new log
08:03:37.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.406481 7349 ts_tablet_manager.cc:1397] T 469e5c7686c0457dab3668241a8bd9e0 P ed40a832914747859ad22b79871417e6: Time spent bootstrapping tablet: real 0.008s user 0.000s sys 0.005s
08:03:37.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.407270 7348 tablet_bootstrap.cc:654] T 469e5c7686c0457dab3668241a8bd9e0 P f7b24a2ea005466d8b25624cc1a9044f: Neither blocks nor log segments found. Creating new log.
08:03:37.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.407292 7350 tablet_bootstrap.cc:492] T 469e5c7686c0457dab3668241a8bd9e0 P 090753c2c8b748e9a1d89d70c8001722: No bootstrap required, opened a new log
08:03:37.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.407549 7350 ts_tablet_manager.cc:1397] T 469e5c7686c0457dab3668241a8bd9e0 P 090753c2c8b748e9a1d89d70c8001722: Time spent bootstrapping tablet: real 0.007s user 0.000s sys 0.004s
08:03:37.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.408248 7348 log.cc:826] T 469e5c7686c0457dab3668241a8bd9e0 P f7b24a2ea005466d8b25624cc1a9044f: Log is configured to *not* fsync() on all Append() calls
08:03:37.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.410594 7348 tablet_bootstrap.cc:492] T 469e5c7686c0457dab3668241a8bd9e0 P f7b24a2ea005466d8b25624cc1a9044f: No bootstrap required, opened a new log
08:03:37.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.410912 7348 ts_tablet_manager.cc:1397] T 469e5c7686c0457dab3668241a8bd9e0 P f7b24a2ea005466d8b25624cc1a9044f: Time spent bootstrapping tablet: real 0.007s user 0.003s sys 0.003s
08:03:37.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.413573 7349 raft_consensus.cc:357] T 469e5c7686c0457dab3668241a8bd9e0 P ed40a832914747859ad22b79871417e6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7b24a2ea005466d8b25624cc1a9044f" member_type: VOTER last_known_addr { host: "127.6.111.193" port: 36349 } } peers { permanent_uuid: "090753c2c8b748e9a1d89d70c8001722" member_type: VOTER last_known_addr { host: "127.6.111.194" port: 44167 } } peers { permanent_uuid: "ed40a832914747859ad22b79871417e6" member_type: VOTER last_known_addr { host: "127.6.111.195" port: 33209 } }
08:03:37.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.414023 7349 raft_consensus.cc:383] T 469e5c7686c0457dab3668241a8bd9e0 P ed40a832914747859ad22b79871417e6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:37.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.414212 7349 raft_consensus.cc:738] T 469e5c7686c0457dab3668241a8bd9e0 P ed40a832914747859ad22b79871417e6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ed40a832914747859ad22b79871417e6, State: Initialized, Role: FOLLOWER
08:03:37.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.414822 7350 raft_consensus.cc:357] T 469e5c7686c0457dab3668241a8bd9e0 P 090753c2c8b748e9a1d89d70c8001722 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7b24a2ea005466d8b25624cc1a9044f" member_type: VOTER last_known_addr { host: "127.6.111.193" port: 36349 } } peers { permanent_uuid: "090753c2c8b748e9a1d89d70c8001722" member_type: VOTER last_known_addr { host: "127.6.111.194" port: 44167 } } peers { permanent_uuid: "ed40a832914747859ad22b79871417e6" member_type: VOTER last_known_addr { host: "127.6.111.195" port: 33209 } }
08:03:37.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.414914 7349 consensus_queue.cc:260] T 469e5c7686c0457dab3668241a8bd9e0 P ed40a832914747859ad22b79871417e6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7b24a2ea005466d8b25624cc1a9044f" member_type: VOTER last_known_addr { host: "127.6.111.193" port: 36349 } } peers { permanent_uuid: "090753c2c8b748e9a1d89d70c8001722" member_type: VOTER last_known_addr { host: "127.6.111.194" port: 44167 } } peers { permanent_uuid: "ed40a832914747859ad22b79871417e6" member_type: VOTER last_known_addr { host: "127.6.111.195" port: 33209 } }
08:03:37.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.415263 7350 raft_consensus.cc:383] T 469e5c7686c0457dab3668241a8bd9e0 P 090753c2c8b748e9a1d89d70c8001722 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:37.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.415513 7350 raft_consensus.cc:738] T 469e5c7686c0457dab3668241a8bd9e0 P 090753c2c8b748e9a1d89d70c8001722 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 090753c2c8b748e9a1d89d70c8001722, State: Initialized, Role: FOLLOWER
08:03:37.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.416388 7350 consensus_queue.cc:260] T 469e5c7686c0457dab3668241a8bd9e0 P 090753c2c8b748e9a1d89d70c8001722 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7b24a2ea005466d8b25624cc1a9044f" member_type: VOTER last_known_addr { host: "127.6.111.193" port: 36349 } } peers { permanent_uuid: "090753c2c8b748e9a1d89d70c8001722" member_type: VOTER last_known_addr { host: "127.6.111.194" port: 44167 } } peers { permanent_uuid: "ed40a832914747859ad22b79871417e6" member_type: VOTER last_known_addr { host: "127.6.111.195" port: 33209 } }
08:03:37.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.419301 7348 raft_consensus.cc:357] T 469e5c7686c0457dab3668241a8bd9e0 P f7b24a2ea005466d8b25624cc1a9044f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7b24a2ea005466d8b25624cc1a9044f" member_type: VOTER last_known_addr { host: "127.6.111.193" port: 36349 } } peers { permanent_uuid: "090753c2c8b748e9a1d89d70c8001722" member_type: VOTER last_known_addr { host: "127.6.111.194" port: 44167 } } peers { permanent_uuid: "ed40a832914747859ad22b79871417e6" member_type: VOTER last_known_addr { host: "127.6.111.195" port: 33209 } }
08:03:37.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.419757 7348 raft_consensus.cc:383] T 469e5c7686c0457dab3668241a8bd9e0 P f7b24a2ea005466d8b25624cc1a9044f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:37.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.420148 7348 raft_consensus.cc:738] T 469e5c7686c0457dab3668241a8bd9e0 P f7b24a2ea005466d8b25624cc1a9044f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f7b24a2ea005466d8b25624cc1a9044f, State: Initialized, Role: FOLLOWER
08:03:37.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.420815 7348 consensus_queue.cc:260] T 469e5c7686c0457dab3668241a8bd9e0 P f7b24a2ea005466d8b25624cc1a9044f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7b24a2ea005466d8b25624cc1a9044f" member_type: VOTER last_known_addr { host: "127.6.111.193" port: 36349 } } peers { permanent_uuid: "090753c2c8b748e9a1d89d70c8001722" member_type: VOTER last_known_addr { host: "127.6.111.194" port: 44167 } } peers { permanent_uuid: "ed40a832914747859ad22b79871417e6" member_type: VOTER last_known_addr { host: "127.6.111.195" port: 33209 } }
08:03:37.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.422520 7323 heartbeater.cc:499] Master 127.6.111.252:40087 was elected leader, sending a full tablet report...
08:03:37.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.423287 7350 ts_tablet_manager.cc:1428] T 469e5c7686c0457dab3668241a8bd9e0 P 090753c2c8b748e9a1d89d70c8001722: Time spent starting tablet: real 0.016s user 0.009s sys 0.006s
08:03:37.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.423295 7349 ts_tablet_manager.cc:1428] T 469e5c7686c0457dab3668241a8bd9e0 P ed40a832914747859ad22b79871417e6: Time spent starting tablet: real 0.017s user 0.013s sys 0.002s
08:03:37.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.423532 7181 heartbeater.cc:499] Master 127.6.111.252:40087 was elected leader, sending a full tablet report...
08:03:37.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.425750 7349 tablet_bootstrap.cc:492] T 7133300aa73e47f6aae2dd4cc4b7a030 P ed40a832914747859ad22b79871417e6: Bootstrap starting.
08:03:37.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.427662 7349 tablet_bootstrap.cc:654] T 7133300aa73e47f6aae2dd4cc4b7a030 P ed40a832914747859ad22b79871417e6: Neither blocks nor log segments found. Creating new log.
08:03:37.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.433605 7350 tablet_bootstrap.cc:492] T 7133300aa73e47f6aae2dd4cc4b7a030 P 090753c2c8b748e9a1d89d70c8001722: Bootstrap starting.
08:03:37.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.436302 7350 tablet_bootstrap.cc:654] T 7133300aa73e47f6aae2dd4cc4b7a030 P 090753c2c8b748e9a1d89d70c8001722: Neither blocks nor log segments found. Creating new log.
08:03:37.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.438572 7350 tablet_bootstrap.cc:492] T 7133300aa73e47f6aae2dd4cc4b7a030 P 090753c2c8b748e9a1d89d70c8001722: No bootstrap required, opened a new log
08:03:37.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.438856 7350 ts_tablet_manager.cc:1397] T 7133300aa73e47f6aae2dd4cc4b7a030 P 090753c2c8b748e9a1d89d70c8001722: Time spent bootstrapping tablet: real 0.005s user 0.004s sys 0.001s
08:03:37.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.439739 7350 raft_consensus.cc:357] T 7133300aa73e47f6aae2dd4cc4b7a030 P 090753c2c8b748e9a1d89d70c8001722 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "090753c2c8b748e9a1d89d70c8001722" member_type: VOTER last_known_addr { host: "127.6.111.194" port: 44167 } } peers { permanent_uuid: "ed40a832914747859ad22b79871417e6" member_type: VOTER last_known_addr { host: "127.6.111.195" port: 33209 } } peers { permanent_uuid: "f7b24a2ea005466d8b25624cc1a9044f" member_type: VOTER last_known_addr { host: "127.6.111.193" port: 36349 } }
08:03:37.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.440096 7350 raft_consensus.cc:383] T 7133300aa73e47f6aae2dd4cc4b7a030 P 090753c2c8b748e9a1d89d70c8001722 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:37.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.440287 7350 raft_consensus.cc:738] T 7133300aa73e47f6aae2dd4cc4b7a030 P 090753c2c8b748e9a1d89d70c8001722 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 090753c2c8b748e9a1d89d70c8001722, State: Initialized, Role: FOLLOWER
08:03:37.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.440655 7350 consensus_queue.cc:260] T 7133300aa73e47f6aae2dd4cc4b7a030 P 090753c2c8b748e9a1d89d70c8001722 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "090753c2c8b748e9a1d89d70c8001722" member_type: VOTER last_known_addr { host: "127.6.111.194" port: 44167 } } peers { permanent_uuid: "ed40a832914747859ad22b79871417e6" member_type: VOTER last_known_addr { host: "127.6.111.195" port: 33209 } } peers { permanent_uuid: "f7b24a2ea005466d8b25624cc1a9044f" member_type: VOTER last_known_addr { host: "127.6.111.193" port: 36349 } }
08:03:37.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.441336 7350 ts_tablet_manager.cc:1428] T 7133300aa73e47f6aae2dd4cc4b7a030 P 090753c2c8b748e9a1d89d70c8001722: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
08:03:37.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.443500 7349 tablet_bootstrap.cc:492] T 7133300aa73e47f6aae2dd4cc4b7a030 P ed40a832914747859ad22b79871417e6: No bootstrap required, opened a new log
08:03:37.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.443758 7349 ts_tablet_manager.cc:1397] T 7133300aa73e47f6aae2dd4cc4b7a030 P ed40a832914747859ad22b79871417e6: Time spent bootstrapping tablet: real 0.018s user 0.007s sys 0.006s
08:03:37.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.444591 7349 raft_consensus.cc:357] T 7133300aa73e47f6aae2dd4cc4b7a030 P ed40a832914747859ad22b79871417e6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "090753c2c8b748e9a1d89d70c8001722" member_type: VOTER last_known_addr { host: "127.6.111.194" port: 44167 } } peers { permanent_uuid: "ed40a832914747859ad22b79871417e6" member_type: VOTER last_known_addr { host: "127.6.111.195" port: 33209 } } peers { permanent_uuid: "f7b24a2ea005466d8b25624cc1a9044f" member_type: VOTER last_known_addr { host: "127.6.111.193" port: 36349 } }
08:03:37.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.444924 7349 raft_consensus.cc:383] T 7133300aa73e47f6aae2dd4cc4b7a030 P ed40a832914747859ad22b79871417e6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:37.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.445063 7349 raft_consensus.cc:738] T 7133300aa73e47f6aae2dd4cc4b7a030 P ed40a832914747859ad22b79871417e6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ed40a832914747859ad22b79871417e6, State: Initialized, Role: FOLLOWER
08:03:37.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.445372 7349 consensus_queue.cc:260] T 7133300aa73e47f6aae2dd4cc4b7a030 P ed40a832914747859ad22b79871417e6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "090753c2c8b748e9a1d89d70c8001722" member_type: VOTER last_known_addr { host: "127.6.111.194" port: 44167 } } peers { permanent_uuid: "ed40a832914747859ad22b79871417e6" member_type: VOTER last_known_addr { host: "127.6.111.195" port: 33209 } } peers { permanent_uuid: "f7b24a2ea005466d8b25624cc1a9044f" member_type: VOTER last_known_addr { host: "127.6.111.193" port: 36349 } }
08:03:37.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.445451 7348 ts_tablet_manager.cc:1428] T 469e5c7686c0457dab3668241a8bd9e0 P f7b24a2ea005466d8b25624cc1a9044f: Time spent starting tablet: real 0.034s user 0.009s sys 0.004s
08:03:37.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.446385 7349 ts_tablet_manager.cc:1428] T 7133300aa73e47f6aae2dd4cc4b7a030 P ed40a832914747859ad22b79871417e6: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
08:03:37.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.450484 7348 tablet_bootstrap.cc:492] T 7133300aa73e47f6aae2dd4cc4b7a030 P f7b24a2ea005466d8b25624cc1a9044f: Bootstrap starting.
08:03:37.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.451884 7348 tablet_bootstrap.cc:654] T 7133300aa73e47f6aae2dd4cc4b7a030 P f7b24a2ea005466d8b25624cc1a9044f: Neither blocks nor log segments found. Creating new log.
08:03:37.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.453390 7348 tablet_bootstrap.cc:492] T 7133300aa73e47f6aae2dd4cc4b7a030 P f7b24a2ea005466d8b25624cc1a9044f: No bootstrap required, opened a new log
08:03:37.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.453557 7348 ts_tablet_manager.cc:1397] T 7133300aa73e47f6aae2dd4cc4b7a030 P f7b24a2ea005466d8b25624cc1a9044f: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
08:03:37.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.454074 7348 raft_consensus.cc:357] T 7133300aa73e47f6aae2dd4cc4b7a030 P f7b24a2ea005466d8b25624cc1a9044f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "090753c2c8b748e9a1d89d70c8001722" member_type: VOTER last_known_addr { host: "127.6.111.194" port: 44167 } } peers { permanent_uuid: "ed40a832914747859ad22b79871417e6" member_type: VOTER last_known_addr { host: "127.6.111.195" port: 33209 } } peers { permanent_uuid: "f7b24a2ea005466d8b25624cc1a9044f" member_type: VOTER last_known_addr { host: "127.6.111.193" port: 36349 } }
08:03:37.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.454222 7348 raft_consensus.cc:383] T 7133300aa73e47f6aae2dd4cc4b7a030 P f7b24a2ea005466d8b25624cc1a9044f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:37.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.454278 7348 raft_consensus.cc:738] T 7133300aa73e47f6aae2dd4cc4b7a030 P f7b24a2ea005466d8b25624cc1a9044f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f7b24a2ea005466d8b25624cc1a9044f, State: Initialized, Role: FOLLOWER
08:03:37.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.454483 7348 consensus_queue.cc:260] T 7133300aa73e47f6aae2dd4cc4b7a030 P f7b24a2ea005466d8b25624cc1a9044f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "090753c2c8b748e9a1d89d70c8001722" member_type: VOTER last_known_addr { host: "127.6.111.194" port: 44167 } } peers { permanent_uuid: "ed40a832914747859ad22b79871417e6" member_type: VOTER last_known_addr { host: "127.6.111.195" port: 33209 } } peers { permanent_uuid: "f7b24a2ea005466d8b25624cc1a9044f" member_type: VOTER last_known_addr { host: "127.6.111.193" port: 36349 } }
08:03:37.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.454890 7348 ts_tablet_manager.cc:1428] T 7133300aa73e47f6aae2dd4cc4b7a030 P f7b24a2ea005466d8b25624cc1a9044f: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
08:03:37.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:37.504469 7184 tablet.cc:2378] T 469e5c7686c0457dab3668241a8bd9e0 P 090753c2c8b748e9a1d89d70c8001722: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:03:37.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:37.507028 7326 tablet.cc:2378] T 469e5c7686c0457dab3668241a8bd9e0 P ed40a832914747859ad22b79871417e6: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:03:37.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:37.538681 7043 tablet.cc:2378] T 469e5c7686c0457dab3668241a8bd9e0 P f7b24a2ea005466d8b25624cc1a9044f: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:03:37.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.550930 7355 raft_consensus.cc:491] T 469e5c7686c0457dab3668241a8bd9e0 P 090753c2c8b748e9a1d89d70c8001722 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:03:37.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.551148 7355 raft_consensus.cc:513] T 469e5c7686c0457dab3668241a8bd9e0 P 090753c2c8b748e9a1d89d70c8001722 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7b24a2ea005466d8b25624cc1a9044f" member_type: VOTER last_known_addr { host: "127.6.111.193" port: 36349 } } peers { permanent_uuid: "090753c2c8b748e9a1d89d70c8001722" member_type: VOTER last_known_addr { host: "127.6.111.194" port: 44167 } } peers { permanent_uuid: "ed40a832914747859ad22b79871417e6" member_type: VOTER last_known_addr { host: "127.6.111.195" port: 33209 } }
08:03:37.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.552748 7355 leader_election.cc:290] T 469e5c7686c0457dab3668241a8bd9e0 P 090753c2c8b748e9a1d89d70c8001722 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f7b24a2ea005466d8b25624cc1a9044f (127.6.111.193:36349), ed40a832914747859ad22b79871417e6 (127.6.111.195:33209)
08:03:37.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.559389 6995 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "469e5c7686c0457dab3668241a8bd9e0" candidate_uuid: "090753c2c8b748e9a1d89d70c8001722" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f7b24a2ea005466d8b25624cc1a9044f" is_pre_election: true
08:03:37.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.559911 6995 raft_consensus.cc:2466] T 469e5c7686c0457dab3668241a8bd9e0 P f7b24a2ea005466d8b25624cc1a9044f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 090753c2c8b748e9a1d89d70c8001722 in term 0.
08:03:37.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.559834 7273 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "469e5c7686c0457dab3668241a8bd9e0" candidate_uuid: "090753c2c8b748e9a1d89d70c8001722" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ed40a832914747859ad22b79871417e6" is_pre_election: true
08:03:37.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.560186 7273 raft_consensus.cc:2466] T 469e5c7686c0457dab3668241a8bd9e0 P ed40a832914747859ad22b79871417e6 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 090753c2c8b748e9a1d89d70c8001722 in term 0.
08:03:37.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.560461 7063 leader_election.cc:304] T 469e5c7686c0457dab3668241a8bd9e0 P 090753c2c8b748e9a1d89d70c8001722 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 090753c2c8b748e9a1d89d70c8001722, f7b24a2ea005466d8b25624cc1a9044f; no voters:
08:03:37.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.560858 7355 raft_consensus.cc:2802] T 469e5c7686c0457dab3668241a8bd9e0 P 090753c2c8b748e9a1d89d70c8001722 [term 0 FOLLOWER]: Leader pre-election won for term 1
08:03:37.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.560998 7355 raft_consensus.cc:491] T 469e5c7686c0457dab3668241a8bd9e0 P 090753c2c8b748e9a1d89d70c8001722 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:03:37.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.561102 7355 raft_consensus.cc:3058] T 469e5c7686c0457dab3668241a8bd9e0 P 090753c2c8b748e9a1d89d70c8001722 [term 0 FOLLOWER]: Advancing to term 1
08:03:37.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.563107 7355 raft_consensus.cc:513] T 469e5c7686c0457dab3668241a8bd9e0 P 090753c2c8b748e9a1d89d70c8001722 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7b24a2ea005466d8b25624cc1a9044f" member_type: VOTER last_known_addr { host: "127.6.111.193" port: 36349 } } peers { permanent_uuid: "090753c2c8b748e9a1d89d70c8001722" member_type: VOTER last_known_addr { host: "127.6.111.194" port: 44167 } } peers { permanent_uuid: "ed40a832914747859ad22b79871417e6" member_type: VOTER last_known_addr { host: "127.6.111.195" port: 33209 } }
08:03:37.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.563650 7355 leader_election.cc:290] T 469e5c7686c0457dab3668241a8bd9e0 P 090753c2c8b748e9a1d89d70c8001722 [CANDIDATE]: Term 1 election: Requested vote from peers f7b24a2ea005466d8b25624cc1a9044f (127.6.111.193:36349), ed40a832914747859ad22b79871417e6 (127.6.111.195:33209)
08:03:37.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.564136 6995 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "469e5c7686c0457dab3668241a8bd9e0" candidate_uuid: "090753c2c8b748e9a1d89d70c8001722" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f7b24a2ea005466d8b25624cc1a9044f"
08:03:37.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.564165 7273 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "469e5c7686c0457dab3668241a8bd9e0" candidate_uuid: "090753c2c8b748e9a1d89d70c8001722" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ed40a832914747859ad22b79871417e6"
08:03:37.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.564297 6995 raft_consensus.cc:3058] T 469e5c7686c0457dab3668241a8bd9e0 P f7b24a2ea005466d8b25624cc1a9044f [term 0 FOLLOWER]: Advancing to term 1
08:03:37.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.564356 7273 raft_consensus.cc:3058] T 469e5c7686c0457dab3668241a8bd9e0 P ed40a832914747859ad22b79871417e6 [term 0 FOLLOWER]: Advancing to term 1
08:03:37.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.566442 6995 raft_consensus.cc:2466] T 469e5c7686c0457dab3668241a8bd9e0 P f7b24a2ea005466d8b25624cc1a9044f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 090753c2c8b748e9a1d89d70c8001722 in term 1.
08:03:37.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.566792 7063 leader_election.cc:304] T 469e5c7686c0457dab3668241a8bd9e0 P 090753c2c8b748e9a1d89d70c8001722 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 090753c2c8b748e9a1d89d70c8001722, f7b24a2ea005466d8b25624cc1a9044f; no voters:
08:03:37.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.567052 7355 raft_consensus.cc:2802] T 469e5c7686c0457dab3668241a8bd9e0 P 090753c2c8b748e9a1d89d70c8001722 [term 1 FOLLOWER]: Leader election won for term 1
08:03:37.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.567556 7273 raft_consensus.cc:2466] T 469e5c7686c0457dab3668241a8bd9e0 P ed40a832914747859ad22b79871417e6 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 090753c2c8b748e9a1d89d70c8001722 in term 1.
08:03:37.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.567546 7355 raft_consensus.cc:695] T 469e5c7686c0457dab3668241a8bd9e0 P 090753c2c8b748e9a1d89d70c8001722 [term 1 LEADER]: Becoming Leader. State: Replica: 090753c2c8b748e9a1d89d70c8001722, State: Running, Role: LEADER
08:03:37.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.568055 7355 consensus_queue.cc:237] T 469e5c7686c0457dab3668241a8bd9e0 P 090753c2c8b748e9a1d89d70c8001722 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7b24a2ea005466d8b25624cc1a9044f" member_type: VOTER last_known_addr { host: "127.6.111.193" port: 36349 } } peers { permanent_uuid: "090753c2c8b748e9a1d89d70c8001722" member_type: VOTER last_known_addr { host: "127.6.111.194" port: 44167 } } peers { permanent_uuid: "ed40a832914747859ad22b79871417e6" member_type: VOTER last_known_addr { host: "127.6.111.195" port: 33209 } }
08:03:37.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.573495 6795 catalog_manager.cc:5582] T 469e5c7686c0457dab3668241a8bd9e0 P 090753c2c8b748e9a1d89d70c8001722 reported cstate change: term changed from 0 to 1, leader changed from <none> to 090753c2c8b748e9a1d89d70c8001722 (127.6.111.194). New cstate: current_term: 1 leader_uuid: "090753c2c8b748e9a1d89d70c8001722" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7b24a2ea005466d8b25624cc1a9044f" member_type: VOTER last_known_addr { host: "127.6.111.193" port: 36349 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "090753c2c8b748e9a1d89d70c8001722" member_type: VOTER last_known_addr { host: "127.6.111.194" port: 44167 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ed40a832914747859ad22b79871417e6" member_type: VOTER last_known_addr { host: "127.6.111.195" port: 33209 } health_report { overall_health: UNKNOWN } } }
08:03:37.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.614079 7354 raft_consensus.cc:491] T 7133300aa73e47f6aae2dd4cc4b7a030 P ed40a832914747859ad22b79871417e6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:03:37.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.614274 7354 raft_consensus.cc:513] T 7133300aa73e47f6aae2dd4cc4b7a030 P ed40a832914747859ad22b79871417e6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "090753c2c8b748e9a1d89d70c8001722" member_type: VOTER last_known_addr { host: "127.6.111.194" port: 44167 } } peers { permanent_uuid: "ed40a832914747859ad22b79871417e6" member_type: VOTER last_known_addr { host: "127.6.111.195" port: 33209 } } peers { permanent_uuid: "f7b24a2ea005466d8b25624cc1a9044f" member_type: VOTER last_known_addr { host: "127.6.111.193" port: 36349 } }
08:03:37.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.615912 7354 leader_election.cc:290] T 7133300aa73e47f6aae2dd4cc4b7a030 P ed40a832914747859ad22b79871417e6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 090753c2c8b748e9a1d89d70c8001722 (127.6.111.194:44167), f7b24a2ea005466d8b25624cc1a9044f (127.6.111.193:36349)
08:03:37.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.621521 6995 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "7133300aa73e47f6aae2dd4cc4b7a030" candidate_uuid: "ed40a832914747859ad22b79871417e6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f7b24a2ea005466d8b25624cc1a9044f" is_pre_election: true
08:03:37.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.621729 6995 raft_consensus.cc:2466] T 7133300aa73e47f6aae2dd4cc4b7a030 P f7b24a2ea005466d8b25624cc1a9044f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ed40a832914747859ad22b79871417e6 in term 0.
08:03:37.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.621618 7136 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "7133300aa73e47f6aae2dd4cc4b7a030" candidate_uuid: "ed40a832914747859ad22b79871417e6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "090753c2c8b748e9a1d89d70c8001722" is_pre_election: true
08:03:37.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.621985 7136 raft_consensus.cc:2466] T 7133300aa73e47f6aae2dd4cc4b7a030 P 090753c2c8b748e9a1d89d70c8001722 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ed40a832914747859ad22b79871417e6 in term 0.
08:03:37.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.622149 7205 leader_election.cc:304] T 7133300aa73e47f6aae2dd4cc4b7a030 P ed40a832914747859ad22b79871417e6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: ed40a832914747859ad22b79871417e6, f7b24a2ea005466d8b25624cc1a9044f; no voters:
08:03:37.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.622493 7354 raft_consensus.cc:2802] T 7133300aa73e47f6aae2dd4cc4b7a030 P ed40a832914747859ad22b79871417e6 [term 0 FOLLOWER]: Leader pre-election won for term 1
08:03:37.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.622625 7354 raft_consensus.cc:491] T 7133300aa73e47f6aae2dd4cc4b7a030 P ed40a832914747859ad22b79871417e6 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:03:37.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.622720 7354 raft_consensus.cc:3058] T 7133300aa73e47f6aae2dd4cc4b7a030 P ed40a832914747859ad22b79871417e6 [term 0 FOLLOWER]: Advancing to term 1
08:03:37.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.624619 7354 raft_consensus.cc:513] T 7133300aa73e47f6aae2dd4cc4b7a030 P ed40a832914747859ad22b79871417e6 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "090753c2c8b748e9a1d89d70c8001722" member_type: VOTER last_known_addr { host: "127.6.111.194" port: 44167 } } peers { permanent_uuid: "ed40a832914747859ad22b79871417e6" member_type: VOTER last_known_addr { host: "127.6.111.195" port: 33209 } } peers { permanent_uuid: "f7b24a2ea005466d8b25624cc1a9044f" member_type: VOTER last_known_addr { host: "127.6.111.193" port: 36349 } }
08:03:37.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.625218 7354 leader_election.cc:290] T 7133300aa73e47f6aae2dd4cc4b7a030 P ed40a832914747859ad22b79871417e6 [CANDIDATE]: Term 1 election: Requested vote from peers 090753c2c8b748e9a1d89d70c8001722 (127.6.111.194:44167), f7b24a2ea005466d8b25624cc1a9044f (127.6.111.193:36349)
08:03:37.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.625587 7136 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "7133300aa73e47f6aae2dd4cc4b7a030" candidate_uuid: "ed40a832914747859ad22b79871417e6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "090753c2c8b748e9a1d89d70c8001722"
08:03:37.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.625670 6995 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "7133300aa73e47f6aae2dd4cc4b7a030" candidate_uuid: "ed40a832914747859ad22b79871417e6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f7b24a2ea005466d8b25624cc1a9044f"
08:03:37.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.625731 7136 raft_consensus.cc:3058] T 7133300aa73e47f6aae2dd4cc4b7a030 P 090753c2c8b748e9a1d89d70c8001722 [term 0 FOLLOWER]: Advancing to term 1
08:03:37.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.625818 6995 raft_consensus.cc:3058] T 7133300aa73e47f6aae2dd4cc4b7a030 P f7b24a2ea005466d8b25624cc1a9044f [term 0 FOLLOWER]: Advancing to term 1
08:03:37.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.627492 6995 raft_consensus.cc:2466] T 7133300aa73e47f6aae2dd4cc4b7a030 P f7b24a2ea005466d8b25624cc1a9044f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ed40a832914747859ad22b79871417e6 in term 1.
08:03:37.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.627584 7136 raft_consensus.cc:2466] T 7133300aa73e47f6aae2dd4cc4b7a030 P 090753c2c8b748e9a1d89d70c8001722 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ed40a832914747859ad22b79871417e6 in term 1.
08:03:37.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.627882 7205 leader_election.cc:304] T 7133300aa73e47f6aae2dd4cc4b7a030 P ed40a832914747859ad22b79871417e6 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: ed40a832914747859ad22b79871417e6, f7b24a2ea005466d8b25624cc1a9044f; no voters:
08:03:37.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.628223 7354 raft_consensus.cc:2802] T 7133300aa73e47f6aae2dd4cc4b7a030 P ed40a832914747859ad22b79871417e6 [term 1 FOLLOWER]: Leader election won for term 1
08:03:37.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.628561 7354 raft_consensus.cc:695] T 7133300aa73e47f6aae2dd4cc4b7a030 P ed40a832914747859ad22b79871417e6 [term 1 LEADER]: Becoming Leader. State: Replica: ed40a832914747859ad22b79871417e6, State: Running, Role: LEADER
08:03:37.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.629509 7354 consensus_queue.cc:237] T 7133300aa73e47f6aae2dd4cc4b7a030 P ed40a832914747859ad22b79871417e6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "090753c2c8b748e9a1d89d70c8001722" member_type: VOTER last_known_addr { host: "127.6.111.194" port: 44167 } } peers { permanent_uuid: "ed40a832914747859ad22b79871417e6" member_type: VOTER last_known_addr { host: "127.6.111.195" port: 33209 } } peers { permanent_uuid: "f7b24a2ea005466d8b25624cc1a9044f" member_type: VOTER last_known_addr { host: "127.6.111.193" port: 36349 } }
08:03:37.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.634732 6795 catalog_manager.cc:5582] T 7133300aa73e47f6aae2dd4cc4b7a030 P ed40a832914747859ad22b79871417e6 reported cstate change: term changed from 0 to 1, leader changed from <none> to ed40a832914747859ad22b79871417e6 (127.6.111.195). New cstate: current_term: 1 leader_uuid: "ed40a832914747859ad22b79871417e6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "090753c2c8b748e9a1d89d70c8001722" member_type: VOTER last_known_addr { host: "127.6.111.194" port: 44167 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ed40a832914747859ad22b79871417e6" member_type: VOTER last_known_addr { host: "127.6.111.195" port: 33209 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f7b24a2ea005466d8b25624cc1a9044f" member_type: VOTER last_known_addr { host: "127.6.111.193" port: 36349 } health_report { overall_health: UNKNOWN } } }
08:03:37.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.705102 6795 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:50552:
08:03:37.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
08:03:37.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
08:03:37.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:37.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
08:03:37.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
08:03:37.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:03:37.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:37.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:03:37.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:03:37.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:37.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:37.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:37.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:37.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
08:03:37.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
08:03:37.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:03:37.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:37.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:03:37.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:03:37.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:37.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:37.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:37.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:37.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
08:03:37.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
08:03:37.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:03:37.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:37.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:03:37.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:03:37.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:37.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:37.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:37.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:37.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
08:03:37.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
08:03:37.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
08:03:37.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:37.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:03:37.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:03:37.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:37.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:37.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:37.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:37.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
08:03:37.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
08:03:37.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
08:03:37.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
08:03:37.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:37.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
08:03:37.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
08:03:37.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:37.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
08:03:37.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:37.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:37.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:37.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
08:03:37.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:37.706457 6795 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
08:03:37.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.719153 7115 tablet_service.cc:1468] Processing CreateTablet for tablet e7dad1e7bb4f41d4bf4edb315538bedf (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=33acefd51ec741f1bb55b2c9623413f3]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
08:03:37.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.720261 7115 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet e7dad1e7bb4f41d4bf4edb315538bedf. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:37.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.720924 7251 tablet_service.cc:1468] Processing CreateTablet for tablet e7dad1e7bb4f41d4bf4edb315538bedf (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=33acefd51ec741f1bb55b2c9623413f3]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
08:03:37.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.721019 6973 tablet_service.cc:1468] Processing CreateTablet for tablet e7dad1e7bb4f41d4bf4edb315538bedf (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=33acefd51ec741f1bb55b2c9623413f3]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
08:03:37.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.721843 7251 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet e7dad1e7bb4f41d4bf4edb315538bedf. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:37.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.721963 6973 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet e7dad1e7bb4f41d4bf4edb315538bedf. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:37.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.727795 7348 tablet_bootstrap.cc:492] T e7dad1e7bb4f41d4bf4edb315538bedf P f7b24a2ea005466d8b25624cc1a9044f: Bootstrap starting.
08:03:37.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.728396 7350 tablet_bootstrap.cc:492] T e7dad1e7bb4f41d4bf4edb315538bedf P 090753c2c8b748e9a1d89d70c8001722: Bootstrap starting.
08:03:37.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.729420 7349 tablet_bootstrap.cc:492] T e7dad1e7bb4f41d4bf4edb315538bedf P ed40a832914747859ad22b79871417e6: Bootstrap starting.
08:03:37.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.729662 7348 tablet_bootstrap.cc:654] T e7dad1e7bb4f41d4bf4edb315538bedf P f7b24a2ea005466d8b25624cc1a9044f: Neither blocks nor log segments found. Creating new log.
08:03:37.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.730137 7350 tablet_bootstrap.cc:654] T e7dad1e7bb4f41d4bf4edb315538bedf P 090753c2c8b748e9a1d89d70c8001722: Neither blocks nor log segments found. Creating new log.
08:03:37.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.731076 7349 tablet_bootstrap.cc:654] T e7dad1e7bb4f41d4bf4edb315538bedf P ed40a832914747859ad22b79871417e6: Neither blocks nor log segments found. Creating new log.
08:03:37.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.732131 7348 tablet_bootstrap.cc:492] T e7dad1e7bb4f41d4bf4edb315538bedf P f7b24a2ea005466d8b25624cc1a9044f: No bootstrap required, opened a new log
08:03:37.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.732292 7348 ts_tablet_manager.cc:1397] T e7dad1e7bb4f41d4bf4edb315538bedf P f7b24a2ea005466d8b25624cc1a9044f: Time spent bootstrapping tablet: real 0.005s user 0.000s sys 0.004s
08:03:37.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.733011 7349 tablet_bootstrap.cc:492] T e7dad1e7bb4f41d4bf4edb315538bedf P ed40a832914747859ad22b79871417e6: No bootstrap required, opened a new log
08:03:37.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.733131 7349 ts_tablet_manager.cc:1397] T e7dad1e7bb4f41d4bf4edb315538bedf P ed40a832914747859ad22b79871417e6: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
08:03:37.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.733259 7348 raft_consensus.cc:357] T e7dad1e7bb4f41d4bf4edb315538bedf P f7b24a2ea005466d8b25624cc1a9044f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7b24a2ea005466d8b25624cc1a9044f" member_type: VOTER last_known_addr { host: "127.6.111.193" port: 36349 } } peers { permanent_uuid: "090753c2c8b748e9a1d89d70c8001722" member_type: VOTER last_known_addr { host: "127.6.111.194" port: 44167 } } peers { permanent_uuid: "ed40a832914747859ad22b79871417e6" member_type: VOTER last_known_addr { host: "127.6.111.195" port: 33209 } }
08:03:37.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.733465 7348 raft_consensus.cc:383] T e7dad1e7bb4f41d4bf4edb315538bedf P f7b24a2ea005466d8b25624cc1a9044f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:37.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.733587 7348 raft_consensus.cc:738] T e7dad1e7bb4f41d4bf4edb315538bedf P f7b24a2ea005466d8b25624cc1a9044f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f7b24a2ea005466d8b25624cc1a9044f, State: Initialized, Role: FOLLOWER
08:03:37.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.733851 7349 raft_consensus.cc:357] T e7dad1e7bb4f41d4bf4edb315538bedf P ed40a832914747859ad22b79871417e6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7b24a2ea005466d8b25624cc1a9044f" member_type: VOTER last_known_addr { host: "127.6.111.193" port: 36349 } } peers { permanent_uuid: "090753c2c8b748e9a1d89d70c8001722" member_type: VOTER last_known_addr { host: "127.6.111.194" port: 44167 } } peers { permanent_uuid: "ed40a832914747859ad22b79871417e6" member_type: VOTER last_known_addr { host: "127.6.111.195" port: 33209 } }
08:03:37.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.734038 7349 raft_consensus.cc:383] T e7dad1e7bb4f41d4bf4edb315538bedf P ed40a832914747859ad22b79871417e6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:37.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.734127 7349 raft_consensus.cc:738] T e7dad1e7bb4f41d4bf4edb315538bedf P ed40a832914747859ad22b79871417e6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ed40a832914747859ad22b79871417e6, State: Initialized, Role: FOLLOWER
08:03:37.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.734426 7348 consensus_queue.cc:260] T e7dad1e7bb4f41d4bf4edb315538bedf P f7b24a2ea005466d8b25624cc1a9044f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7b24a2ea005466d8b25624cc1a9044f" member_type: VOTER last_known_addr { host: "127.6.111.193" port: 36349 } } peers { permanent_uuid: "090753c2c8b748e9a1d89d70c8001722" member_type: VOTER last_known_addr { host: "127.6.111.194" port: 44167 } } peers { permanent_uuid: "ed40a832914747859ad22b79871417e6" member_type: VOTER last_known_addr { host: "127.6.111.195" port: 33209 } }
08:03:37.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.734558 7349 consensus_queue.cc:260] T e7dad1e7bb4f41d4bf4edb315538bedf P ed40a832914747859ad22b79871417e6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7b24a2ea005466d8b25624cc1a9044f" member_type: VOTER last_known_addr { host: "127.6.111.193" port: 36349 } } peers { permanent_uuid: "090753c2c8b748e9a1d89d70c8001722" member_type: VOTER last_known_addr { host: "127.6.111.194" port: 44167 } } peers { permanent_uuid: "ed40a832914747859ad22b79871417e6" member_type: VOTER last_known_addr { host: "127.6.111.195" port: 33209 } }
08:03:37.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.735039 7354 tablet_replica.cc:440] T e7dad1e7bb4f41d4bf4edb315538bedf P ed40a832914747859ad22b79871417e6: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7b24a2ea005466d8b25624cc1a9044f" member_type: VOTER last_known_addr { host: "127.6.111.193" port: 36349 } } peers { permanent_uuid: "090753c2c8b748e9a1d89d70c8001722" member_type: VOTER last_known_addr { host: "127.6.111.194" port: 44167 } } peers { permanent_uuid: "ed40a832914747859ad22b79871417e6" member_type: VOTER last_known_addr { host: "127.6.111.195" port: 33209 } } }
08:03:37.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.735213 7349 ts_tablet_manager.cc:1428] T e7dad1e7bb4f41d4bf4edb315538bedf P ed40a832914747859ad22b79871417e6: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
08:03:37.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.735275 7354 tablet_replica.cc:443] T e7dad1e7bb4f41d4bf4edb315538bedf P ed40a832914747859ad22b79871417e6: This TxnStatusTablet replica's current role is: FOLLOWER
08:03:37.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.735183 7356 tablet_replica.cc:440] T e7dad1e7bb4f41d4bf4edb315538bedf P f7b24a2ea005466d8b25624cc1a9044f: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7b24a2ea005466d8b25624cc1a9044f" member_type: VOTER last_known_addr { host: "127.6.111.193" port: 36349 } } peers { permanent_uuid: "090753c2c8b748e9a1d89d70c8001722" member_type: VOTER last_known_addr { host: "127.6.111.194" port: 44167 } } peers { permanent_uuid: "ed40a832914747859ad22b79871417e6" member_type: VOTER last_known_addr { host: "127.6.111.195" port: 33209 } } }
08:03:37.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.735436 7356 tablet_replica.cc:443] T e7dad1e7bb4f41d4bf4edb315538bedf P f7b24a2ea005466d8b25624cc1a9044f: This TxnStatusTablet replica's current role is: FOLLOWER
08:03:37.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.739151 7348 ts_tablet_manager.cc:1428] T e7dad1e7bb4f41d4bf4edb315538bedf P f7b24a2ea005466d8b25624cc1a9044f: Time spent starting tablet: real 0.004s user 0.002s sys 0.000s
08:03:37.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.746915 7350 tablet_bootstrap.cc:492] T e7dad1e7bb4f41d4bf4edb315538bedf P 090753c2c8b748e9a1d89d70c8001722: No bootstrap required, opened a new log
08:03:37.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.747058 7350 ts_tablet_manager.cc:1397] T e7dad1e7bb4f41d4bf4edb315538bedf P 090753c2c8b748e9a1d89d70c8001722: Time spent bootstrapping tablet: real 0.019s user 0.006s sys 0.004s
08:03:37.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.747795 7350 raft_consensus.cc:357] T e7dad1e7bb4f41d4bf4edb315538bedf P 090753c2c8b748e9a1d89d70c8001722 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7b24a2ea005466d8b25624cc1a9044f" member_type: VOTER last_known_addr { host: "127.6.111.193" port: 36349 } } peers { permanent_uuid: "090753c2c8b748e9a1d89d70c8001722" member_type: VOTER last_known_addr { host: "127.6.111.194" port: 44167 } } peers { permanent_uuid: "ed40a832914747859ad22b79871417e6" member_type: VOTER last_known_addr { host: "127.6.111.195" port: 33209 } }
08:03:37.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.747952 7350 raft_consensus.cc:383] T e7dad1e7bb4f41d4bf4edb315538bedf P 090753c2c8b748e9a1d89d70c8001722 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:37.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.748039 7350 raft_consensus.cc:738] T e7dad1e7bb4f41d4bf4edb315538bedf P 090753c2c8b748e9a1d89d70c8001722 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 090753c2c8b748e9a1d89d70c8001722, State: Initialized, Role: FOLLOWER
08:03:37.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.748277 7350 consensus_queue.cc:260] T e7dad1e7bb4f41d4bf4edb315538bedf P 090753c2c8b748e9a1d89d70c8001722 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7b24a2ea005466d8b25624cc1a9044f" member_type: VOTER last_known_addr { host: "127.6.111.193" port: 36349 } } peers { permanent_uuid: "090753c2c8b748e9a1d89d70c8001722" member_type: VOTER last_known_addr { host: "127.6.111.194" port: 44167 } } peers { permanent_uuid: "ed40a832914747859ad22b79871417e6" member_type: VOTER last_known_addr { host: "127.6.111.195" port: 33209 } }
08:03:37.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.748822 7355 tablet_replica.cc:440] T e7dad1e7bb4f41d4bf4edb315538bedf P 090753c2c8b748e9a1d89d70c8001722: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7b24a2ea005466d8b25624cc1a9044f" member_type: VOTER last_known_addr { host: "127.6.111.193" port: 36349 } } peers { permanent_uuid: "090753c2c8b748e9a1d89d70c8001722" member_type: VOTER last_known_addr { host: "127.6.111.194" port: 44167 } } peers { permanent_uuid: "ed40a832914747859ad22b79871417e6" member_type: VOTER last_known_addr { host: "127.6.111.195" port: 33209 } } }
08:03:37.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.749075 7355 tablet_replica.cc:443] T e7dad1e7bb4f41d4bf4edb315538bedf P 090753c2c8b748e9a1d89d70c8001722: This TxnStatusTablet replica's current role is: FOLLOWER
08:03:37.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.749876 7350 ts_tablet_manager.cc:1428] T e7dad1e7bb4f41d4bf4edb315538bedf P 090753c2c8b748e9a1d89d70c8001722: Time spent starting tablet: real 0.003s user 0.001s sys 0.001s
08:03:37.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.753149 7356 raft_consensus.cc:491] T e7dad1e7bb4f41d4bf4edb315538bedf P f7b24a2ea005466d8b25624cc1a9044f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:03:37.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.753338 7356 raft_consensus.cc:513] T e7dad1e7bb4f41d4bf4edb315538bedf P f7b24a2ea005466d8b25624cc1a9044f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7b24a2ea005466d8b25624cc1a9044f" member_type: VOTER last_known_addr { host: "127.6.111.193" port: 36349 } } peers { permanent_uuid: "090753c2c8b748e9a1d89d70c8001722" member_type: VOTER last_known_addr { host: "127.6.111.194" port: 44167 } } peers { permanent_uuid: "ed40a832914747859ad22b79871417e6" member_type: VOTER last_known_addr { host: "127.6.111.195" port: 33209 } }
08:03:37.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.755098 7356 leader_election.cc:290] T e7dad1e7bb4f41d4bf4edb315538bedf P f7b24a2ea005466d8b25624cc1a9044f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 090753c2c8b748e9a1d89d70c8001722 (127.6.111.194:44167), ed40a832914747859ad22b79871417e6 (127.6.111.195:33209)
08:03:37.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.757486 7355 raft_consensus.cc:491] T e7dad1e7bb4f41d4bf4edb315538bedf P 090753c2c8b748e9a1d89d70c8001722 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:03:37.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.757642 7355 raft_consensus.cc:513] T e7dad1e7bb4f41d4bf4edb315538bedf P 090753c2c8b748e9a1d89d70c8001722 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7b24a2ea005466d8b25624cc1a9044f" member_type: VOTER last_known_addr { host: "127.6.111.193" port: 36349 } } peers { permanent_uuid: "090753c2c8b748e9a1d89d70c8001722" member_type: VOTER last_known_addr { host: "127.6.111.194" port: 44167 } } peers { permanent_uuid: "ed40a832914747859ad22b79871417e6" member_type: VOTER last_known_addr { host: "127.6.111.195" port: 33209 } }
08:03:37.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.758462 7355 leader_election.cc:290] T e7dad1e7bb4f41d4bf4edb315538bedf P 090753c2c8b748e9a1d89d70c8001722 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f7b24a2ea005466d8b25624cc1a9044f (127.6.111.193:36349), ed40a832914747859ad22b79871417e6 (127.6.111.195:33209)
08:03:37.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.758878 7273 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "e7dad1e7bb4f41d4bf4edb315538bedf" candidate_uuid: "090753c2c8b748e9a1d89d70c8001722" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ed40a832914747859ad22b79871417e6" is_pre_election: true
08:03:37.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.759083 7273 raft_consensus.cc:2466] T e7dad1e7bb4f41d4bf4edb315538bedf P ed40a832914747859ad22b79871417e6 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 090753c2c8b748e9a1d89d70c8001722 in term 0.
08:03:37.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.760993 7063 leader_election.cc:304] T e7dad1e7bb4f41d4bf4edb315538bedf P 090753c2c8b748e9a1d89d70c8001722 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 090753c2c8b748e9a1d89d70c8001722, ed40a832914747859ad22b79871417e6; no voters:
08:03:37.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.761245 7355 raft_consensus.cc:2802] T e7dad1e7bb4f41d4bf4edb315538bedf P 090753c2c8b748e9a1d89d70c8001722 [term 0 FOLLOWER]: Leader pre-election won for term 1
08:03:37.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.761343 7355 raft_consensus.cc:491] T e7dad1e7bb4f41d4bf4edb315538bedf P 090753c2c8b748e9a1d89d70c8001722 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:03:37.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.761449 7355 raft_consensus.cc:3058] T e7dad1e7bb4f41d4bf4edb315538bedf P 090753c2c8b748e9a1d89d70c8001722 [term 0 FOLLOWER]: Advancing to term 1
08:03:37.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.761704 6995 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "e7dad1e7bb4f41d4bf4edb315538bedf" candidate_uuid: "090753c2c8b748e9a1d89d70c8001722" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f7b24a2ea005466d8b25624cc1a9044f" is_pre_election: true
08:03:37.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.761862 6995 raft_consensus.cc:2466] T e7dad1e7bb4f41d4bf4edb315538bedf P f7b24a2ea005466d8b25624cc1a9044f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 090753c2c8b748e9a1d89d70c8001722 in term 0.
08:03:37.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.764544 7355 raft_consensus.cc:513] T e7dad1e7bb4f41d4bf4edb315538bedf P 090753c2c8b748e9a1d89d70c8001722 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7b24a2ea005466d8b25624cc1a9044f" member_type: VOTER last_known_addr { host: "127.6.111.193" port: 36349 } } peers { permanent_uuid: "090753c2c8b748e9a1d89d70c8001722" member_type: VOTER last_known_addr { host: "127.6.111.194" port: 44167 } } peers { permanent_uuid: "ed40a832914747859ad22b79871417e6" member_type: VOTER last_known_addr { host: "127.6.111.195" port: 33209 } }
08:03:37.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.765206 7355 leader_election.cc:290] T e7dad1e7bb4f41d4bf4edb315538bedf P 090753c2c8b748e9a1d89d70c8001722 [CANDIDATE]: Term 1 election: Requested vote from peers f7b24a2ea005466d8b25624cc1a9044f (127.6.111.193:36349), ed40a832914747859ad22b79871417e6 (127.6.111.195:33209)
08:03:37.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.765641 7273 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "e7dad1e7bb4f41d4bf4edb315538bedf" candidate_uuid: "090753c2c8b748e9a1d89d70c8001722" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ed40a832914747859ad22b79871417e6"
08:03:37.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.765854 7273 raft_consensus.cc:3058] T e7dad1e7bb4f41d4bf4edb315538bedf P ed40a832914747859ad22b79871417e6 [term 0 FOLLOWER]: Advancing to term 1
08:03:37.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.768913 7273 raft_consensus.cc:2466] T e7dad1e7bb4f41d4bf4edb315538bedf P ed40a832914747859ad22b79871417e6 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 090753c2c8b748e9a1d89d70c8001722 in term 1.
08:03:37.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.769306 7063 leader_election.cc:304] T e7dad1e7bb4f41d4bf4edb315538bedf P 090753c2c8b748e9a1d89d70c8001722 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 090753c2c8b748e9a1d89d70c8001722, ed40a832914747859ad22b79871417e6; no voters:
08:03:37.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.769589 7355 raft_consensus.cc:2802] T e7dad1e7bb4f41d4bf4edb315538bedf P 090753c2c8b748e9a1d89d70c8001722 [term 1 FOLLOWER]: Leader election won for term 1
08:03:37.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.769778 7355 raft_consensus.cc:695] T e7dad1e7bb4f41d4bf4edb315538bedf P 090753c2c8b748e9a1d89d70c8001722 [term 1 LEADER]: Becoming Leader. State: Replica: 090753c2c8b748e9a1d89d70c8001722, State: Running, Role: LEADER
08:03:37.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.769966 6995 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "e7dad1e7bb4f41d4bf4edb315538bedf" candidate_uuid: "090753c2c8b748e9a1d89d70c8001722" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f7b24a2ea005466d8b25624cc1a9044f"
08:03:37.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.770067 7355 consensus_queue.cc:237] T e7dad1e7bb4f41d4bf4edb315538bedf P 090753c2c8b748e9a1d89d70c8001722 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7b24a2ea005466d8b25624cc1a9044f" member_type: VOTER last_known_addr { host: "127.6.111.193" port: 36349 } } peers { permanent_uuid: "090753c2c8b748e9a1d89d70c8001722" member_type: VOTER last_known_addr { host: "127.6.111.194" port: 44167 } } peers { permanent_uuid: "ed40a832914747859ad22b79871417e6" member_type: VOTER last_known_addr { host: "127.6.111.195" port: 33209 } }
08:03:37.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.770165 6995 raft_consensus.cc:3058] T e7dad1e7bb4f41d4bf4edb315538bedf P f7b24a2ea005466d8b25624cc1a9044f [term 0 FOLLOWER]: Advancing to term 1
08:03:37.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.770347 7273 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "e7dad1e7bb4f41d4bf4edb315538bedf" candidate_uuid: "f7b24a2ea005466d8b25624cc1a9044f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ed40a832914747859ad22b79871417e6" is_pre_election: true
08:03:37.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.770642 7273 raft_consensus.cc:2391] T e7dad1e7bb4f41d4bf4edb315538bedf P ed40a832914747859ad22b79871417e6 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate f7b24a2ea005466d8b25624cc1a9044f in current term 1: Already voted for candidate 090753c2c8b748e9a1d89d70c8001722 in this term.
08:03:37.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.770366 7136 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "e7dad1e7bb4f41d4bf4edb315538bedf" candidate_uuid: "f7b24a2ea005466d8b25624cc1a9044f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "090753c2c8b748e9a1d89d70c8001722" is_pre_election: true
08:03:37.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.772260 6918 leader_election.cc:304] T e7dad1e7bb4f41d4bf4edb315538bedf P f7b24a2ea005466d8b25624cc1a9044f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: f7b24a2ea005466d8b25624cc1a9044f; no voters: 090753c2c8b748e9a1d89d70c8001722, ed40a832914747859ad22b79871417e6
08:03:37.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.772850 6995 raft_consensus.cc:2466] T e7dad1e7bb4f41d4bf4edb315538bedf P f7b24a2ea005466d8b25624cc1a9044f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 090753c2c8b748e9a1d89d70c8001722 in term 1.
08:03:37.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.773146 7356 raft_consensus.cc:2747] T e7dad1e7bb4f41d4bf4edb315538bedf P f7b24a2ea005466d8b25624cc1a9044f [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
08:03:37.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.774144 7362 tablet_replica.cc:440] T e7dad1e7bb4f41d4bf4edb315538bedf P 090753c2c8b748e9a1d89d70c8001722: TxnStatusTablet state changed. Reason: New leader 090753c2c8b748e9a1d89d70c8001722. Latest consensus state: current_term: 1 leader_uuid: "090753c2c8b748e9a1d89d70c8001722" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7b24a2ea005466d8b25624cc1a9044f" member_type: VOTER last_known_addr { host: "127.6.111.193" port: 36349 } } peers { permanent_uuid: "090753c2c8b748e9a1d89d70c8001722" member_type: VOTER last_known_addr { host: "127.6.111.194" port: 44167 } } peers { permanent_uuid: "ed40a832914747859ad22b79871417e6" member_type: VOTER last_known_addr { host: "127.6.111.195" port: 33209 } } }
08:03:37.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.774427 7362 tablet_replica.cc:443] T e7dad1e7bb4f41d4bf4edb315538bedf P 090753c2c8b748e9a1d89d70c8001722: This TxnStatusTablet replica's current role is: LEADER
08:03:37.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.775137 7381 txn_status_manager.cc:874] Waiting until node catch up with all replicated operations in previous term...
08:03:37.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.775455 7381 txn_status_manager.cc:930] Loading transaction status metadata into memory...
08:03:37.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.776355 6795 catalog_manager.cc:5582] T e7dad1e7bb4f41d4bf4edb315538bedf P 090753c2c8b748e9a1d89d70c8001722 reported cstate change: term changed from 0 to 1, leader changed from <none> to 090753c2c8b748e9a1d89d70c8001722 (127.6.111.194). New cstate: current_term: 1 leader_uuid: "090753c2c8b748e9a1d89d70c8001722" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7b24a2ea005466d8b25624cc1a9044f" member_type: VOTER last_known_addr { host: "127.6.111.193" port: 36349 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "090753c2c8b748e9a1d89d70c8001722" member_type: VOTER last_known_addr { host: "127.6.111.194" port: 44167 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ed40a832914747859ad22b79871417e6" member_type: VOTER last_known_addr { host: "127.6.111.195" port: 33209 } health_report { overall_health: UNKNOWN } } }
08:03:37.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.813436 7273 raft_consensus.cc:1273] T e7dad1e7bb4f41d4bf4edb315538bedf P ed40a832914747859ad22b79871417e6 [term 1 FOLLOWER]: Refusing update from remote peer 090753c2c8b748e9a1d89d70c8001722: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:03:37.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.814011 6995 raft_consensus.cc:1273] T e7dad1e7bb4f41d4bf4edb315538bedf P f7b24a2ea005466d8b25624cc1a9044f [term 1 FOLLOWER]: Refusing update from remote peer 090753c2c8b748e9a1d89d70c8001722: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:03:37.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.814249 7355 consensus_queue.cc:1035] T e7dad1e7bb4f41d4bf4edb315538bedf P 090753c2c8b748e9a1d89d70c8001722 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ed40a832914747859ad22b79871417e6" member_type: VOTER last_known_addr { host: "127.6.111.195" port: 33209 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:03:37.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.814777 7362 consensus_queue.cc:1035] T e7dad1e7bb4f41d4bf4edb315538bedf P 090753c2c8b748e9a1d89d70c8001722 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f7b24a2ea005466d8b25624cc1a9044f" member_type: VOTER last_known_addr { host: "127.6.111.193" port: 36349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:03:37.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.831416 7356 tablet_replica.cc:440] T e7dad1e7bb4f41d4bf4edb315538bedf P f7b24a2ea005466d8b25624cc1a9044f: TxnStatusTablet state changed. Reason: New leader 090753c2c8b748e9a1d89d70c8001722. Latest consensus state: current_term: 1 leader_uuid: "090753c2c8b748e9a1d89d70c8001722" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7b24a2ea005466d8b25624cc1a9044f" member_type: VOTER last_known_addr { host: "127.6.111.193" port: 36349 } } peers { permanent_uuid: "090753c2c8b748e9a1d89d70c8001722" member_type: VOTER last_known_addr { host: "127.6.111.194" port: 44167 } } peers { permanent_uuid: "ed40a832914747859ad22b79871417e6" member_type: VOTER last_known_addr { host: "127.6.111.195" port: 33209 } } }
08:03:37.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.831691 7356 tablet_replica.cc:443] T e7dad1e7bb4f41d4bf4edb315538bedf P f7b24a2ea005466d8b25624cc1a9044f: This TxnStatusTablet replica's current role is: FOLLOWER
08:03:37.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.832130 7354 tablet_replica.cc:440] T e7dad1e7bb4f41d4bf4edb315538bedf P ed40a832914747859ad22b79871417e6: TxnStatusTablet state changed. Reason: New leader 090753c2c8b748e9a1d89d70c8001722. Latest consensus state: current_term: 1 leader_uuid: "090753c2c8b748e9a1d89d70c8001722" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7b24a2ea005466d8b25624cc1a9044f" member_type: VOTER last_known_addr { host: "127.6.111.193" port: 36349 } } peers { permanent_uuid: "090753c2c8b748e9a1d89d70c8001722" member_type: VOTER last_known_addr { host: "127.6.111.194" port: 44167 } } peers { permanent_uuid: "ed40a832914747859ad22b79871417e6" member_type: VOTER last_known_addr { host: "127.6.111.195" port: 33209 } } }
08:03:37.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.832350 7354 tablet_replica.cc:443] T e7dad1e7bb4f41d4bf4edb315538bedf P ed40a832914747859ad22b79871417e6: This TxnStatusTablet replica's current role is: FOLLOWER
08:03:37.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.835947 7362 tablet_replica.cc:440] T e7dad1e7bb4f41d4bf4edb315538bedf P 090753c2c8b748e9a1d89d70c8001722: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "090753c2c8b748e9a1d89d70c8001722" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7b24a2ea005466d8b25624cc1a9044f" member_type: VOTER last_known_addr { host: "127.6.111.193" port: 36349 } } peers { permanent_uuid: "090753c2c8b748e9a1d89d70c8001722" member_type: VOTER last_known_addr { host: "127.6.111.194" port: 44167 } } peers { permanent_uuid: "ed40a832914747859ad22b79871417e6" member_type: VOTER last_known_addr { host: "127.6.111.195" port: 33209 } } }
08:03:37.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.836158 7362 tablet_replica.cc:443] T e7dad1e7bb4f41d4bf4edb315538bedf P 090753c2c8b748e9a1d89d70c8001722: This TxnStatusTablet replica's current role is: LEADER
08:03:37.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.838460 7385 mvcc.cc:204] Tried to move back new op lower bound from 7171434978554318848 to 7171434978393853952. Current Snapshot: MvccSnapshot[applied={T|T < 7171434978554318848}]
08:03:37.841 [DEBUG - Time-limited test] (KuduTransaction.java:702) starting keepalive heartbeating with period 500 ms (txn ID 0)
08:03:37.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.841874 7355 tablet_replica.cc:440] T e7dad1e7bb4f41d4bf4edb315538bedf P 090753c2c8b748e9a1d89d70c8001722: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "090753c2c8b748e9a1d89d70c8001722" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7b24a2ea005466d8b25624cc1a9044f" member_type: VOTER last_known_addr { host: "127.6.111.193" port: 36349 } } peers { permanent_uuid: "090753c2c8b748e9a1d89d70c8001722" member_type: VOTER last_known_addr { host: "127.6.111.194" port: 44167 } } peers { permanent_uuid: "ed40a832914747859ad22b79871417e6" member_type: VOTER last_known_addr { host: "127.6.111.195" port: 33209 } } }
08:03:37.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.842120 7355 tablet_replica.cc:443] T e7dad1e7bb4f41d4bf4edb315538bedf P 090753c2c8b748e9a1d89d70c8001722: This TxnStatusTablet replica's current role is: LEADER
08:03:37.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.842242 7386 mvcc.cc:204] Tried to move back new op lower bound from 7171434978554318848 to 7171434978393853952. Current Snapshot: MvccSnapshot[applied={T|T < 7171434978554318848}]
08:03:37.844 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table e46ea1d2e29e4e89869fdbc2f3d47cdc
08:03:37.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.847249 7356 tablet_replica.cc:440] T e7dad1e7bb4f41d4bf4edb315538bedf P f7b24a2ea005466d8b25624cc1a9044f: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "090753c2c8b748e9a1d89d70c8001722" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7b24a2ea005466d8b25624cc1a9044f" member_type: VOTER last_known_addr { host: "127.6.111.193" port: 36349 } } peers { permanent_uuid: "090753c2c8b748e9a1d89d70c8001722" member_type: VOTER last_known_addr { host: "127.6.111.194" port: 44167 } } peers { permanent_uuid: "ed40a832914747859ad22b79871417e6" member_type: VOTER last_known_addr { host: "127.6.111.195" port: 33209 } } }
08:03:37.847 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 7133300aa73e47f6aae2dd4cc4b7a030 for table 'txn_manager_fallback' with partition [0x00000000, 0x00000001)
08:03:37.847 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 469e5c7686c0457dab3668241a8bd9e0 for table 'txn_manager_fallback' with partition [0x00000001, 0x00000002)
08:03:37.848 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x00000000, upperBoundPartitionKey=0x00000001, ttl=299999, tablet=7133300aa73e47f6aae2dd4cc4b7a030@[090753c2c8b748e9a1d89d70c8001722(127.6.111.194:44167),ed40a832914747859ad22b79871417e6(127.6.111.195:33209)[L],f7b24a2ea005466d8b25624cc1a9044f(127.6.111.193:36349)]}, Tablet{lowerBoundPartitionKey=0x00000001, upperBoundPartitionKey=0x00000002, ttl=299999, tablet=469e5c7686c0457dab3668241a8bd9e0@[090753c2c8b748e9a1d89d70c8001722(127.6.111.194:44167)[L],ed40a832914747859ad22b79871417e6(127.6.111.195:33209),f7b24a2ea005466d8b25624cc1a9044f(127.6.111.193:36349)]}, NonCoveredRange{lowerBoundPartitionKey=0x00000002, upperBoundPartitionKey=0x, ttl=299999}]
08:03:37.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.847469 7356 tablet_replica.cc:443] T e7dad1e7bb4f41d4bf4edb315538bedf P f7b24a2ea005466d8b25624cc1a9044f: This TxnStatusTablet replica's current role is: FOLLOWER
08:03:37.848 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=4), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.111.252:40087, [3ms] Sub RPC GetTableLocations: received response from server master-127.6.111.252:40087: OK, deferred=null) after lookup
08:03:37.850 [DEBUG - kudu-nio-0] (Connection.java:748) [peer ed40a832914747859ad22b79871417e6(127.6.111.195:33209)] connecting to peer
08:03:37.852 [DEBUG - kudu-nio-1] (Connection.java:757) [peer ed40a832914747859ad22b79871417e6(127.6.111.195:33209)] Successfully connected to peer
08:03:37.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.854065 7354 tablet_replica.cc:440] T e7dad1e7bb4f41d4bf4edb315538bedf P ed40a832914747859ad22b79871417e6: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "090753c2c8b748e9a1d89d70c8001722" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7b24a2ea005466d8b25624cc1a9044f" member_type: VOTER last_known_addr { host: "127.6.111.193" port: 36349 } } peers { permanent_uuid: "090753c2c8b748e9a1d89d70c8001722" member_type: VOTER last_known_addr { host: "127.6.111.194" port: 44167 } } peers { permanent_uuid: "ed40a832914747859ad22b79871417e6" member_type: VOTER last_known_addr { host: "127.6.111.195" port: 33209 } } }
08:03:37.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.854301 7354 tablet_replica.cc:443] T e7dad1e7bb4f41d4bf4edb315538bedf P ed40a832914747859ad22b79871417e6: This TxnStatusTablet replica's current role is: FOLLOWER
08:03:37.864 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x01c29f8d, L:/127.0.0.1:43146 - R:/127.6.111.195:33209] using TOKEN/null
08:03:37.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.891431 6995 raft_consensus.cc:1273] T 7133300aa73e47f6aae2dd4cc4b7a030 P f7b24a2ea005466d8b25624cc1a9044f [term 1 FOLLOWER]: Refusing update from remote peer ed40a832914747859ad22b79871417e6: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:03:37.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.891762 7136 raft_consensus.cc:1273] T 7133300aa73e47f6aae2dd4cc4b7a030 P 090753c2c8b748e9a1d89d70c8001722 [term 1 FOLLOWER]: Refusing update from remote peer ed40a832914747859ad22b79871417e6: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:03:37.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.892292 7354 consensus_queue.cc:1035] T 7133300aa73e47f6aae2dd4cc4b7a030 P ed40a832914747859ad22b79871417e6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "090753c2c8b748e9a1d89d70c8001722" member_type: VOTER last_known_addr { host: "127.6.111.194" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:03:37.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.892541 7365 consensus_queue.cc:1035] T 7133300aa73e47f6aae2dd4cc4b7a030 P ed40a832914747859ad22b79871417e6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f7b24a2ea005466d8b25624cc1a9044f" member_type: VOTER last_known_addr { host: "127.6.111.193" port: 36349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:03:37.919 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.6.111.252:40087
08:03:37.919 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
08:03:37.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.922492 6591 external_mini_cluster.cc:1661] Attempting to check leaks for /tmp/dist-test-task8V1MUJ/build/asan/bin/kudu pid 6761
08:03:37.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:37.923844 6835 generic_service.cc:196] Checking for leaks (request via RPC)
08:03:38.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.038558 6591 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task8V1MUJ/build/asan/bin/kudu with pid 6761
08:03:38.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.041913 7390 consensus_queue.cc:1035] T 469e5c7686c0457dab3668241a8bd9e0 P 090753c2c8b748e9a1d89d70c8001722 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ed40a832914747859ad22b79871417e6" member_type: VOTER last_known_addr { host: "127.6.111.195" port: 33209 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:03:38.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.049629 7390 consensus_queue.cc:1035] T 469e5c7686c0457dab3668241a8bd9e0 P 090753c2c8b748e9a1d89d70c8001722 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f7b24a2ea005466d8b25624cc1a9044f" member_type: VOTER last_known_addr { host: "127.6.111.193" port: 36349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:03:38.070 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.6.111.252:40087(127.6.111.252:40087)] handling channelInactive
08:03:38.071 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.111.252:40087(127.6.111.252:40087)] cleaning up while in state READY due to: connection closed
08:03:38.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:38.071311 7204 connection.cc:537] client connection to 127.6.111.252:40087 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
08:03:38.072 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:03:38.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:38.072223 7062 connection.cc:537] client connection to 127.6.111.252:40087 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
08:03:38.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:38.072635 7181 heartbeater.cc:646] Failed to heartbeat to 127.6.111.252:40087 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv error from unknown peer: Transport endpoint is not connected (error 107)
08:03:38.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:38.073551 6618 connection.cc:537] server connection from 127.0.0.1:51178 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
08:03:38.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:38.073834 6700 connection.cc:537] server connection from 127.0.0.1:60702 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
08:03:38.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:38.074812 7323 heartbeater.cc:646] Failed to heartbeat to 127.6.111.252:40087 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv error from unknown peer: Transport endpoint is not connected (error 107)
08:03:38.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:38.076346 6918 connection.cc:537] client connection to 127.6.111.252:40087 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
08:03:38.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:38.076953 7040 heartbeater.cc:646] Failed to heartbeat to 127.6.111.252:40087 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv error from unknown peer: Transport endpoint is not connected (error 107)
08:03:38.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.339745 7403 raft_consensus.cc:491] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 883f8f327b0e4b72b943de5ef25445f2)
08:03:38.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.339890 7404 raft_consensus.cc:491] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 883f8f327b0e4b72b943de5ef25445f2)
08:03:38.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.339949 7403 raft_consensus.cc:513] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "37df082dc4d74885bb14a018ac6c52a1" member_type: VOTER last_known_addr { host: "127.6.111.254" port: 46321 } } peers { permanent_uuid: "98a4571016524600ae1e653f157c03c0" member_type: VOTER last_known_addr { host: "127.6.111.253" port: 39259 } } peers { permanent_uuid: "883f8f327b0e4b72b943de5ef25445f2" member_type: VOTER last_known_addr { host: "127.6.111.252" port: 40087 } }
08:03:38.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.340031 7404 raft_consensus.cc:513] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "37df082dc4d74885bb14a018ac6c52a1" member_type: VOTER last_known_addr { host: "127.6.111.254" port: 46321 } } peers { permanent_uuid: "98a4571016524600ae1e653f157c03c0" member_type: VOTER last_known_addr { host: "127.6.111.253" port: 39259 } } peers { permanent_uuid: "883f8f327b0e4b72b943de5ef25445f2" member_type: VOTER last_known_addr { host: "127.6.111.252" port: 40087 } }
08:03:38.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.341240 7403 leader_election.cc:290] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 37df082dc4d74885bb14a018ac6c52a1 (127.6.111.254:46321), 883f8f327b0e4b72b943de5ef25445f2 (127.6.111.252:40087)
08:03:38.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.341251 7404 leader_election.cc:290] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 98a4571016524600ae1e653f157c03c0 (127.6.111.253:39259), 883f8f327b0e4b72b943de5ef25445f2 (127.6.111.252:40087)
08:03:38.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.341734 6722 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "37df082dc4d74885bb14a018ac6c52a1" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "98a4571016524600ae1e653f157c03c0" is_pre_election: true
08:03:38.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.341753 6642 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "98a4571016524600ae1e653f157c03c0" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "37df082dc4d74885bb14a018ac6c52a1" is_pre_election: true
08:03:38.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.341948 6642 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 98a4571016524600ae1e653f157c03c0 in term 1.
08:03:38.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.341955 6722 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 37df082dc4d74885bb14a018ac6c52a1 in term 1.
08:03:38.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.342429 6620 leader_election.cc:304] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 37df082dc4d74885bb14a018ac6c52a1, 98a4571016524600ae1e653f157c03c0; no voters:
08:03:38.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.342922 7404 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [term 1 FOLLOWER]: Leader pre-election won for term 2
08:03:38.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.343158 7404 raft_consensus.cc:491] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 883f8f327b0e4b72b943de5ef25445f2)
08:03:38.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.343343 7404 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [term 1 FOLLOWER]: Advancing to term 2
08:03:38.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.343650 6700 leader_election.cc:304] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 37df082dc4d74885bb14a018ac6c52a1, 98a4571016524600ae1e653f157c03c0; no voters:
08:03:38.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.343989 7403 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0 [term 1 FOLLOWER]: Leader pre-election won for term 2
08:03:38.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.344098 7403 raft_consensus.cc:491] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 883f8f327b0e4b72b943de5ef25445f2)
08:03:38.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.344200 7403 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0 [term 1 FOLLOWER]: Advancing to term 2
08:03:38.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:38.345319 6700 leader_election.cc:336] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 883f8f327b0e4b72b943de5ef25445f2 (127.6.111.252:40087): Network error: Client connection negotiation failed: client connection to 127.6.111.252:40087: connect: Connection refused (error 111)
08:03:38.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:38.345490 6619 leader_election.cc:336] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 883f8f327b0e4b72b943de5ef25445f2 (127.6.111.252:40087): Network error: Client connection negotiation failed: client connection to 127.6.111.252:40087: connect: Connection refused (error 111)
08:03:38.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.346508 7404 raft_consensus.cc:513] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "37df082dc4d74885bb14a018ac6c52a1" member_type: VOTER last_known_addr { host: "127.6.111.254" port: 46321 } } peers { permanent_uuid: "98a4571016524600ae1e653f157c03c0" member_type: VOTER last_known_addr { host: "127.6.111.253" port: 39259 } } peers { permanent_uuid: "883f8f327b0e4b72b943de5ef25445f2" member_type: VOTER last_known_addr { host: "127.6.111.252" port: 40087 } }
08:03:38.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.346805 7403 raft_consensus.cc:513] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "37df082dc4d74885bb14a018ac6c52a1" member_type: VOTER last_known_addr { host: "127.6.111.254" port: 46321 } } peers { permanent_uuid: "98a4571016524600ae1e653f157c03c0" member_type: VOTER last_known_addr { host: "127.6.111.253" port: 39259 } } peers { permanent_uuid: "883f8f327b0e4b72b943de5ef25445f2" member_type: VOTER last_known_addr { host: "127.6.111.252" port: 40087 } }
08:03:38.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.347079 7404 leader_election.cc:290] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [CANDIDATE]: Term 2 election: Requested vote from peers 98a4571016524600ae1e653f157c03c0 (127.6.111.253:39259), 883f8f327b0e4b72b943de5ef25445f2 (127.6.111.252:40087)
08:03:38.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.347430 7403 leader_election.cc:290] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0 [CANDIDATE]: Term 2 election: Requested vote from peers 37df082dc4d74885bb14a018ac6c52a1 (127.6.111.254:46321), 883f8f327b0e4b72b943de5ef25445f2 (127.6.111.252:40087)
08:03:38.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.347435 6722 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "37df082dc4d74885bb14a018ac6c52a1" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "98a4571016524600ae1e653f157c03c0"
08:03:38.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.347754 6722 raft_consensus.cc:2391] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 37df082dc4d74885bb14a018ac6c52a1 in current term 2: Already voted for candidate 98a4571016524600ae1e653f157c03c0 in this term.
08:03:38.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.347906 6642 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "98a4571016524600ae1e653f157c03c0" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "37df082dc4d74885bb14a018ac6c52a1"
08:03:38.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.348145 6642 raft_consensus.cc:2391] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 98a4571016524600ae1e653f157c03c0 in current term 2: Already voted for candidate 37df082dc4d74885bb14a018ac6c52a1 in this term.
08:03:38.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:38.348767 6619 leader_election.cc:336] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 883f8f327b0e4b72b943de5ef25445f2 (127.6.111.252:40087): Network error: Client connection negotiation failed: client connection to 127.6.111.252:40087: connect: Connection refused (error 111)
08:03:38.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.348905 6619 leader_election.cc:304] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 37df082dc4d74885bb14a018ac6c52a1; no voters: 883f8f327b0e4b72b943de5ef25445f2, 98a4571016524600ae1e653f157c03c0
08:03:38.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:38.349108 6700 leader_election.cc:336] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 883f8f327b0e4b72b943de5ef25445f2 (127.6.111.252:40087): Network error: Client connection negotiation failed: client connection to 127.6.111.252:40087: connect: Connection refused (error 111)
08:03:38.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.349164 7404 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
08:03:38.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.349224 6700 leader_election.cc:304] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 98a4571016524600ae1e653f157c03c0; no voters: 37df082dc4d74885bb14a018ac6c52a1, 883f8f327b0e4b72b943de5ef25445f2
08:03:38.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.349503 7403 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
08:03:38.360 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.6.111.252:40087(127.6.111.252:40087)] connecting to peer
08:03:38.361 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.361 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.111.252:40087(127.6.111.252:40087)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.361 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location master-127.6.111.252:40087(127.6.111.252:40087) for tablet Kudu Master: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.378 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.6.111.252:40087(127.6.111.252:40087)] connecting to peer
08:03:38.379 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.379 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.111.252:40087(127.6.111.252:40087)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.379 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.111.252:40087: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.380 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321; will retry
08:03:38.397 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.6.111.252:40087(127.6.111.252:40087)] connecting to peer
08:03:38.398 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.399 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.111.252:40087(127.6.111.252:40087)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.399 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.111.252:40087: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.400 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321; will retry
08:03:38.417 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.6.111.252:40087(127.6.111.252:40087)] connecting to peer
08:03:38.419 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.419 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.111.252:40087(127.6.111.252:40087)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.419 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.111.252:40087: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.420 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321; will retry
08:03:38.438 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.6.111.252:40087(127.6.111.252:40087)] connecting to peer
08:03:38.438 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.439 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.111.252:40087(127.6.111.252:40087)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.439 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.111.252:40087: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.439 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321; will retry
08:03:38.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.6.111.252:40087(127.6.111.252:40087)] connecting to peer
08:03:38.458 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.458 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.111.252:40087(127.6.111.252:40087)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.459 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.111.252:40087: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.459 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321; will retry
08:03:38.497 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.6.111.252:40087(127.6.111.252:40087)] connecting to peer
08:03:38.498 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.499 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.111.252:40087(127.6.111.252:40087)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.499 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.111.252:40087: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.500 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321; will retry
08:03:38.558 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.6.111.252:40087(127.6.111.252:40087)] connecting to peer
08:03:38.559 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.559 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.111.252:40087(127.6.111.252:40087)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.559 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.111.252:40087: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.560 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321; will retry
08:03:38.561 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] sending RPC to server master-127.6.111.252:40087, [4ms] received response from server master-127.6.111.252:40087: Network error: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [4ms] delaying RPC due to: Network error: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.252:40087, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.253:39259, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.254:46321, [22ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.252:40087: Network error: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [23ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.253:39259: OK, [23ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.254:46321: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.252:40087, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.253:39259, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.254:46321, [42ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.252:40087: Network error: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [42ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.254:46321: OK, [43ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.253:39259: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.252:40087, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.253:39259, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.254:46321, [62ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.252:40087: Network error: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [63ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.253:39259: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.254:46321: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.252:40087, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.253:39259, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.254:46321, [82ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.252:40087: Network error: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [82ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.254:46321: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.253:39259: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.252:40087, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.253:39259, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.254:46321, [101ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.252:40087: Network error: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [102ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.253:39259: OK, [102ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.254:46321: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.252:40087, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.253:39259, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.254:46321, [142ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.252:40087: Network error: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [142ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.253:39259: OK, [142ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.254:46321: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.252:40087, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.253:39259, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.254:46321, [202ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.252:40087: Network error: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [202ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.253:39259: OK, [202ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.254:46321: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, deferred=Deferred@874033162(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$119/1430840707@378794ab -> (continuation of Deferred@279206938 after retry RPC after error@717086675) -> (continuation of Deferred@1532277358 after retry RPC after error@1350387555) -> (continuation of Deferred@1653315066 after retry RPC after error@646749503) -> (continuation of Deferred@1479908418 after retry RPC after error@1648987768) -> (continuation of Deferred@1549001614 after retry RPC after error@510978142) -> (continuation of Deferred@1784131678 after retry RPC after error@1145297186), errback=org.apache.kudu.client.KuduTransaction$$Lambda$120/558273874@7cf1db7 -> (continuation of Deferred@279206938 after retry RPC after error@717086675) -> (continuation of Deferred@1532277358 after retry RPC after error@1350387555) -> (continuation of Deferred@1653315066 after retry RPC after error@646749503) -> (continuation of Deferred@1479908418 after retry RPC after error@1648987768) -> (continuation of Deferred@1549001614 after retry RPC after error@510978142) -> (continuation of Deferred@1784131678 after retry RPC after error@1145297186)))
08:03:38.561 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] sending RPC to server master-127.6.111.252:40087, [4ms] received response from server master-127.6.111.252:40087: Network error: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [4ms] delaying RPC due to: Network error: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.252:40087, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.253:39259, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.254:46321, [22ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.252:40087: Network error: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [23ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.253:39259: OK, [23ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.254:46321: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.252:40087, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.253:39259, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.254:46321, [42ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.252:40087: Network error: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [42ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.254:46321: OK, [43ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.253:39259: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.252:40087, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.253:39259, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.254:46321, [62ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.252:40087: Network error: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [63ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.253:39259: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.254:46321: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.252:40087, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.253:39259, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.254:46321, [82ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.252:40087: Network error: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [82ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.254:46321: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.253:39259: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.252:40087, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.253:39259, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.254:46321, [101ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.252:40087: Network error: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [102ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.253:39259: OK, [102ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.254:46321: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.252:40087, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.253:39259, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.254:46321, [142ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.252:40087: Network error: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [142ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.253:39259: OK, [142ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.254:46321: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.252:40087, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.253:39259, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.254:46321, [202ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.252:40087: Network error: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [202ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.253:39259: OK, [202ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.254:46321: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, deferred=Deferred@874033162(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$119/1430840707@378794ab -> (continuation of Deferred@279206938 after retry RPC after error@717086675) -> (continuation of Deferred@1532277358 after retry RPC after error@1350387555) -> (continuation of Deferred@1653315066 after retry RPC after error@646749503) -> (continuation of Deferred@1479908418 after retry RPC after error@1648987768) -> (continuation of Deferred@1549001614 after retry RPC after error@510978142) -> (continuation of Deferred@1784131678 after retry RPC after error@1145297186), errback=org.apache.kudu.client.KuduTransaction$$Lambda$120/558273874@7cf1db7 -> (continuation of Deferred@279206938 after retry RPC after error@717086675) -> (continuation of Deferred@1532277358 after retry RPC after error@1350387555) -> (continuation of Deferred@1653315066 after retry RPC after error@646749503) -> (continuation of Deferred@1479908418 after retry RPC after error@1648987768) -> (continuation of Deferred@1549001614 after retry RPC after error@510978142) -> (continuation of Deferred@1784131678 after retry RPC after error@1145297186)))
08:03:38.577 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.6.111.252:40087(127.6.111.252:40087)] connecting to peer
08:03:38.580 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.580 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.111.252:40087(127.6.111.252:40087)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.580 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.111.252:40087: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.580 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321; will retry
08:03:38.597 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.6.111.252:40087(127.6.111.252:40087)] connecting to peer
08:03:38.599 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.599 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.111.252:40087(127.6.111.252:40087)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.599 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.111.252:40087: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.600 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321; will retry
08:03:38.617 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.6.111.252:40087(127.6.111.252:40087)] connecting to peer
08:03:38.619 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.619 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.111.252:40087(127.6.111.252:40087)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.619 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.111.252:40087: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.620 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321; will retry
08:03:38.637 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.6.111.252:40087(127.6.111.252:40087)] connecting to peer
08:03:38.638 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.638 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.111.252:40087(127.6.111.252:40087)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.638 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.111.252:40087: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.639 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321; will retry
08:03:38.657 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.6.111.252:40087(127.6.111.252:40087)] connecting to peer
08:03:38.658 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.658 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.111.252:40087(127.6.111.252:40087)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.658 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.111.252:40087: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.659 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321; will retry
08:03:38.697 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.6.111.252:40087(127.6.111.252:40087)] connecting to peer
08:03:38.698 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.698 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.111.252:40087(127.6.111.252:40087)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.698 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.111.252:40087: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.699 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321; will retry
08:03:38.757 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.6.111.252:40087(127.6.111.252:40087)] connecting to peer
08:03:38.759 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.759 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.111.252:40087(127.6.111.252:40087)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.759 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.111.252:40087: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.761 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321; will retry
08:03:38.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.780870 7403 raft_consensus.cc:491] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:03:38.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.781059 7403 raft_consensus.cc:513] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "37df082dc4d74885bb14a018ac6c52a1" member_type: VOTER last_known_addr { host: "127.6.111.254" port: 46321 } } peers { permanent_uuid: "98a4571016524600ae1e653f157c03c0" member_type: VOTER last_known_addr { host: "127.6.111.253" port: 39259 } } peers { permanent_uuid: "883f8f327b0e4b72b943de5ef25445f2" member_type: VOTER last_known_addr { host: "127.6.111.252" port: 40087 } }
08:03:38.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.781562 7403 leader_election.cc:290] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 37df082dc4d74885bb14a018ac6c52a1 (127.6.111.254:46321), 883f8f327b0e4b72b943de5ef25445f2 (127.6.111.252:40087)
08:03:38.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.782073 6642 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "98a4571016524600ae1e653f157c03c0" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "37df082dc4d74885bb14a018ac6c52a1" is_pre_election: true
08:03:38.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.782259 7404 raft_consensus.cc:491] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:03:38.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.782455 7404 raft_consensus.cc:513] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "37df082dc4d74885bb14a018ac6c52a1" member_type: VOTER last_known_addr { host: "127.6.111.254" port: 46321 } } peers { permanent_uuid: "98a4571016524600ae1e653f157c03c0" member_type: VOTER last_known_addr { host: "127.6.111.253" port: 39259 } } peers { permanent_uuid: "883f8f327b0e4b72b943de5ef25445f2" member_type: VOTER last_known_addr { host: "127.6.111.252" port: 40087 } }
08:03:38.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.782857 6642 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 98a4571016524600ae1e653f157c03c0 in term 2.
08:03:38.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.783155 7404 leader_election.cc:290] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 98a4571016524600ae1e653f157c03c0 (127.6.111.253:39259), 883f8f327b0e4b72b943de5ef25445f2 (127.6.111.252:40087)
08:03:38.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:38.783480 6700 leader_election.cc:336] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 883f8f327b0e4b72b943de5ef25445f2 (127.6.111.252:40087): Network error: Client connection negotiation failed: client connection to 127.6.111.252:40087: connect: Connection refused (error 111)
08:03:38.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.783667 6722 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "37df082dc4d74885bb14a018ac6c52a1" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "98a4571016524600ae1e653f157c03c0" is_pre_election: true
08:03:38.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.783775 6700 leader_election.cc:304] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 37df082dc4d74885bb14a018ac6c52a1, 98a4571016524600ae1e653f157c03c0; no voters: 883f8f327b0e4b72b943de5ef25445f2
08:03:38.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.783901 6722 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 37df082dc4d74885bb14a018ac6c52a1 in term 2.
08:03:38.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.784198 7403 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0 [term 2 FOLLOWER]: Leader pre-election won for term 3
08:03:38.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.784288 7403 raft_consensus.cc:491] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:03:38.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.784404 7403 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0 [term 2 FOLLOWER]: Advancing to term 3
08:03:38.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.784344 6620 leader_election.cc:304] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 37df082dc4d74885bb14a018ac6c52a1, 98a4571016524600ae1e653f157c03c0; no voters:
08:03:38.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.784713 7404 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [term 2 FOLLOWER]: Leader pre-election won for term 3
08:03:38.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:38.784731 6619 leader_election.cc:336] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 883f8f327b0e4b72b943de5ef25445f2 (127.6.111.252:40087): Network error: Client connection negotiation failed: client connection to 127.6.111.252:40087: connect: Connection refused (error 111)
08:03:38.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.784823 7404 raft_consensus.cc:491] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:03:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.784927 7404 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [term 2 FOLLOWER]: Advancing to term 3
08:03:38.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.786727 7403 raft_consensus.cc:513] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "37df082dc4d74885bb14a018ac6c52a1" member_type: VOTER last_known_addr { host: "127.6.111.254" port: 46321 } } peers { permanent_uuid: "98a4571016524600ae1e653f157c03c0" member_type: VOTER last_known_addr { host: "127.6.111.253" port: 39259 } } peers { permanent_uuid: "883f8f327b0e4b72b943de5ef25445f2" member_type: VOTER last_known_addr { host: "127.6.111.252" port: 40087 } }
08:03:38.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.786841 7404 raft_consensus.cc:513] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "37df082dc4d74885bb14a018ac6c52a1" member_type: VOTER last_known_addr { host: "127.6.111.254" port: 46321 } } peers { permanent_uuid: "98a4571016524600ae1e653f157c03c0" member_type: VOTER last_known_addr { host: "127.6.111.253" port: 39259 } } peers { permanent_uuid: "883f8f327b0e4b72b943de5ef25445f2" member_type: VOTER last_known_addr { host: "127.6.111.252" port: 40087 } }
08:03:38.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.787285 7403 leader_election.cc:290] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0 [CANDIDATE]: Term 3 election: Requested vote from peers 37df082dc4d74885bb14a018ac6c52a1 (127.6.111.254:46321), 883f8f327b0e4b72b943de5ef25445f2 (127.6.111.252:40087)
08:03:38.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.787334 7404 leader_election.cc:290] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [CANDIDATE]: Term 3 election: Requested vote from peers 98a4571016524600ae1e653f157c03c0 (127.6.111.253:39259), 883f8f327b0e4b72b943de5ef25445f2 (127.6.111.252:40087)
08:03:38.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.787822 6722 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "37df082dc4d74885bb14a018ac6c52a1" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "98a4571016524600ae1e653f157c03c0"
08:03:38.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.787856 6642 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "98a4571016524600ae1e653f157c03c0" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "37df082dc4d74885bb14a018ac6c52a1"
08:03:38.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.788057 6642 raft_consensus.cc:2391] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [term 3 FOLLOWER]: Leader election vote request: Denying vote to candidate 98a4571016524600ae1e653f157c03c0 in current term 3: Already voted for candidate 37df082dc4d74885bb14a018ac6c52a1 in this term.
08:03:38.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.788072 6722 raft_consensus.cc:2391] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0 [term 3 FOLLOWER]: Leader election vote request: Denying vote to candidate 37df082dc4d74885bb14a018ac6c52a1 in current term 3: Already voted for candidate 98a4571016524600ae1e653f157c03c0 in this term.
08:03:38.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:38.788888 6700 leader_election.cc:336] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 883f8f327b0e4b72b943de5ef25445f2 (127.6.111.252:40087): Network error: Client connection negotiation failed: client connection to 127.6.111.252:40087: connect: Connection refused (error 111)
08:03:38.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:38.788931 6619 leader_election.cc:336] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 883f8f327b0e4b72b943de5ef25445f2 (127.6.111.252:40087): Network error: Client connection negotiation failed: client connection to 127.6.111.252:40087: connect: Connection refused (error 111)
08:03:38.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.789012 6700 leader_election.cc:304] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0 [CANDIDATE]: Term 3 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 98a4571016524600ae1e653f157c03c0; no voters: 37df082dc4d74885bb14a018ac6c52a1, 883f8f327b0e4b72b943de5ef25445f2
08:03:38.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.789031 6619 leader_election.cc:304] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [CANDIDATE]: Term 3 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 37df082dc4d74885bb14a018ac6c52a1; no voters: 883f8f327b0e4b72b943de5ef25445f2, 98a4571016524600ae1e653f157c03c0
08:03:38.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.789209 7403 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0 [term 3 FOLLOWER]: Leader election lost for term 3. Reason: could not achieve majority
08:03:38.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.789253 7404 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [term 3 FOLLOWER]: Leader election lost for term 3. Reason: could not achieve majority
08:03:38.797 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.6.111.252:40087(127.6.111.252:40087)] connecting to peer
08:03:38.798 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.799 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.111.252:40087(127.6.111.252:40087)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.799 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.111.252:40087: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.799 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321; will retry
08:03:38.800 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.252:40087, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.253:39259, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.254:46321, [2ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.253:39259: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.254:46321: OK, [3ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.252:40087: Network error: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [3ms] delaying RPC due to: Service unavailable: Master config (127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.252:40087, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.253:39259, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.254:46321, [22ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.252:40087: Network error: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [22ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.254:46321: OK, [23ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.253:39259: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.252:40087, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.253:39259, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.254:46321, [42ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.252:40087: Network error: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [42ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.253:39259: OK, [43ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.254:46321: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.252:40087, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.253:39259, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.254:46321, [61ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.252:40087: Network error: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [62ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.253:39259: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.254:46321: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.252:40087, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.253:39259, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.254:46321, [81ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.252:40087: Network error: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [82ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.254:46321: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.253:39259: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.252:40087, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.253:39259, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.254:46321, [121ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.252:40087: Network error: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [122ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.254:46321: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.253:39259: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.252:40087, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.253:39259, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.254:46321, [182ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.252:40087: Network error: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [182ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.253:39259: OK, [184ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.254:46321: OK, [184ms] delaying RPC due to: Service unavailable: Master config (127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.252:40087, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.253:39259, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.254:46321, [222ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.252:40087: Network error: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [222ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.254:46321: OK, [222ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.253:39259: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, deferred=Deferred@698644397(state=PENDING, result=null, callback=(continuation of Deferred@1619944971 after retry RPC after error@2120116165) -> (continuation of Deferred@169292151 after retry RPC after error@272007) -> (continuation of Deferred@321500850 after retry RPC after error@507812701) -> (continuation of Deferred@770411883 after retry RPC after error@209364559) -> (continuation of Deferred@1387793748 after retry RPC after error@1746729517) -> (continuation of Deferred@435008144 after retry RPC after error@1090271036) -> (continuation of Deferred@1379684570 after retry RPC after error@1576932642), errback=(continuation of Deferred@1619944971 after retry RPC after error@2120116165) -> (continuation of Deferred@169292151 after retry RPC after error@272007) -> (continuation of Deferred@321500850 after retry RPC after error@507812701) -> (continuation of Deferred@770411883 after retry RPC after error@209364559) -> (continuation of Deferred@1387793748 after retry RPC after error@1746729517) -> (continuation of Deferred@435008144 after retry RPC after error@1090271036) -> (continuation of Deferred@1379684570 after retry RPC after error@1576932642)))
08:03:38.800 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.252:40087, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.253:39259, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.254:46321, [2ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.253:39259: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.254:46321: OK, [3ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.252:40087: Network error: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [3ms] delaying RPC due to: Service unavailable: Master config (127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.252:40087, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.253:39259, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.254:46321, [22ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.252:40087: Network error: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [22ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.254:46321: OK, [23ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.253:39259: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.252:40087, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.253:39259, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.254:46321, [42ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.252:40087: Network error: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [42ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.253:39259: OK, [43ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.254:46321: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.252:40087, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.253:39259, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.254:46321, [61ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.252:40087: Network error: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [62ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.253:39259: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.254:46321: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.252:40087, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.253:39259, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.254:46321, [81ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.252:40087: Network error: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [82ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.254:46321: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.253:39259: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.252:40087, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.253:39259, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.254:46321, [121ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.252:40087: Network error: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [122ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.254:46321: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.253:39259: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.252:40087, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.253:39259, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.254:46321, [182ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.252:40087: Network error: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [182ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.253:39259: OK, [184ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.254:46321: OK, [184ms] delaying RPC due to: Service unavailable: Master config (127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.252:40087, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.253:39259, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.254:46321, [222ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.252:40087: Network error: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [222ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.254:46321: OK, [222ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.253:39259: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, deferred=Deferred@698644397(state=PENDING, result=null, callback=(continuation of Deferred@1619944971 after retry RPC after error@2120116165) -> (continuation of Deferred@169292151 after retry RPC after error@272007) -> (continuation of Deferred@321500850 after retry RPC after error@507812701) -> (continuation of Deferred@770411883 after retry RPC after error@209364559) -> (continuation of Deferred@1387793748 after retry RPC after error@1746729517) -> (continuation of Deferred@435008144 after retry RPC after error@1090271036) -> (continuation of Deferred@1379684570 after retry RPC after error@1576932642), errback=(continuation of Deferred@1619944971 after retry RPC after error@2120116165) -> (continuation of Deferred@169292151 after retry RPC after error@272007) -> (continuation of Deferred@321500850 after retry RPC after error@507812701) -> (continuation of Deferred@770411883 after retry RPC after error@209364559) -> (continuation of Deferred@1387793748 after retry RPC after error@1746729517) -> (continuation of Deferred@435008144 after retry RPC after error@1090271036) -> (continuation of Deferred@1379684570 after retry RPC after error@1576932642)))
08:03:38.817 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.6.111.252:40087(127.6.111.252:40087)] connecting to peer
08:03:38.818 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.818 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.111.252:40087(127.6.111.252:40087)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.818 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.111.252:40087: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.819 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321; will retry
08:03:38.838 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.6.111.252:40087(127.6.111.252:40087)] connecting to peer
08:03:38.839 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.839 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.111.252:40087(127.6.111.252:40087)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.839 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.111.252:40087: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.840 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321; will retry
08:03:38.857 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.6.111.252:40087(127.6.111.252:40087)] connecting to peer
08:03:38.858 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.858 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.111.252:40087(127.6.111.252:40087)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.858 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.111.252:40087: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.859 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321; will retry
08:03:38.877 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.6.111.252:40087(127.6.111.252:40087)] connecting to peer
08:03:38.878 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.879 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.111.252:40087(127.6.111.252:40087)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.879 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.111.252:40087: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.879 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321; will retry
08:03:38.897 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.6.111.252:40087(127.6.111.252:40087)] connecting to peer
08:03:38.898 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.898 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.111.252:40087(127.6.111.252:40087)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.898 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.111.252:40087: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.899 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321; will retry
08:03:38.918 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.6.111.252:40087(127.6.111.252:40087)] connecting to peer
08:03:38.919 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.919 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.111.252:40087(127.6.111.252:40087)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.919 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.111.252:40087: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.920 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321; will retry
08:03:38.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:38.951012 7066 txn_status_manager.cc:1391] automatically aborted stale txn (ID 0) past 1.104s from last keepalive heartbeat (effective timeout is 1.000s)
08:03:38.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:38.953083 7093 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.6.111.252:40087)
08:03:38.998 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.6.111.252:40087(127.6.111.252:40087)] connecting to peer
08:03:38.999 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.999 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.111.252:40087(127.6.111.252:40087)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:38.999 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.111.252:40087: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:39.000 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321; will retry
08:03:39.057 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.6.111.252:40087(127.6.111.252:40087)] connecting to peer
08:03:39.058 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:39.058 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.111.252:40087(127.6.111.252:40087)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:39.058 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.111.252:40087: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:39.059 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321; will retry
08:03:39.060 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.252:40087, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.253:39259, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.254:46321, [1ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.252:40087: Network error: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [1ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.253:39259: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.254:46321: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.252:40087, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.253:39259, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.254:46321, [22ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.252:40087: Network error: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [22ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.253:39259: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.254:46321: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.252:40087, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.253:39259, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.254:46321, [41ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.252:40087: Network error: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [42ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.254:46321: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.253:39259: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.252:40087, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.253:39259, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.254:46321, [62ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.252:40087: Network error: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [62ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.254:46321: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.253:39259: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.252:40087, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.253:39259, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.254:46321, [81ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.252:40087: Network error: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [81ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.253:39259: OK, [81ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.254:46321: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.252:40087, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.253:39259, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.254:46321, [102ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.252:40087: Network error: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [102ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.254:46321: OK, [102ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.253:39259: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.252:40087, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.253:39259, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.254:46321, [182ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.252:40087: Network error: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [182ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.253:39259: OK, [183ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.254:46321: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.252:40087, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.253:39259, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.254:46321, [241ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.252:40087: Network error: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [241ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.254:46321: OK, [242ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.253:39259: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, deferred=Deferred@990065891(state=PENDING, result=null, callback=(continuation of Deferred@1838231637 after retry RPC after error@1437644966) -> (continuation of Deferred@493967571 after retry RPC after error@1073202098) -> (continuation of Deferred@1535139713 after retry RPC after error@195410399) -> (continuation of Deferred@470147000 after retry RPC after error@502167001) -> (continuation of Deferred@1145785778 after retry RPC after error@1652977573) -> (continuation of Deferred@1355584594 after retry RPC after error@1046635311) -> (continuation of Deferred@542805900 after retry RPC after error@1903175525), errback=(continuation of Deferred@1838231637 after retry RPC after error@1437644966) -> (continuation of Deferred@493967571 after retry RPC after error@1073202098) -> (continuation of Deferred@1535139713 after retry RPC after error@195410399) -> (continuation of Deferred@470147000 after retry RPC after error@502167001) -> (continuation of Deferred@1145785778 after retry RPC after error@1652977573) -> (continuation of Deferred@1355584594 after retry RPC after error@1046635311) -> (continuation of Deferred@542805900 after retry RPC after error@1903175525)))
08:03:39.060 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.252:40087, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.253:39259, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.254:46321, [1ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.252:40087: Network error: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [1ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.253:39259: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.254:46321: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.252:40087, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.253:39259, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.254:46321, [22ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.252:40087: Network error: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [22ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.253:39259: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.254:46321: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.252:40087, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.253:39259, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.254:46321, [41ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.252:40087: Network error: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [42ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.254:46321: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.253:39259: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.252:40087, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.253:39259, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.254:46321, [62ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.252:40087: Network error: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [62ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.254:46321: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.253:39259: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.252:40087, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.253:39259, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.254:46321, [81ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.252:40087: Network error: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [81ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.253:39259: OK, [81ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.254:46321: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.252:40087, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.253:39259, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.254:46321, [102ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.252:40087: Network error: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [102ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.254:46321: OK, [102ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.253:39259: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.252:40087, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.253:39259, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.254:46321, [182ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.252:40087: Network error: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [182ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.253:39259: OK, [183ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.254:46321: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.252:40087, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.253:39259, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.254:46321, [241ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.252:40087: Network error: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [241ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.254:46321: OK, [242ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.253:39259: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, deferred=Deferred@990065891(state=PENDING, result=null, callback=(continuation of Deferred@1838231637 after retry RPC after error@1437644966) -> (continuation of Deferred@493967571 after retry RPC after error@1073202098) -> (continuation of Deferred@1535139713 after retry RPC after error@195410399) -> (continuation of Deferred@470147000 after retry RPC after error@502167001) -> (continuation of Deferred@1145785778 after retry RPC after error@1652977573) -> (continuation of Deferred@1355584594 after retry RPC after error@1046635311) -> (continuation of Deferred@542805900 after retry RPC after error@1903175525), errback=(continuation of Deferred@1838231637 after retry RPC after error@1437644966) -> (continuation of Deferred@493967571 after retry RPC after error@1073202098) -> (continuation of Deferred@1535139713 after retry RPC after error@195410399) -> (continuation of Deferred@470147000 after retry RPC after error@502167001) -> (continuation of Deferred@1145785778 after retry RPC after error@1652977573) -> (continuation of Deferred@1355584594 after retry RPC after error@1046635311) -> (continuation of Deferred@542805900 after retry RPC after error@1903175525)))
08:03:39.078 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.6.111.252:40087(127.6.111.252:40087)] connecting to peer
08:03:39.079 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:39.079 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.111.252:40087(127.6.111.252:40087)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:39.079 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.111.252:40087: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:39.083 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321; will retry
08:03:39.097 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.6.111.252:40087(127.6.111.252:40087)] connecting to peer
08:03:39.098 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:39.098 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.111.252:40087(127.6.111.252:40087)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:39.098 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.111.252:40087: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:39.099 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321; will retry
08:03:39.126 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.6.111.252:40087(127.6.111.252:40087)] connecting to peer
08:03:39.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:39.119251 7404 raft_consensus.cc:491] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:03:39.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:39.119423 7404 raft_consensus.cc:513] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "37df082dc4d74885bb14a018ac6c52a1" member_type: VOTER last_known_addr { host: "127.6.111.254" port: 46321 } } peers { permanent_uuid: "98a4571016524600ae1e653f157c03c0" member_type: VOTER last_known_addr { host: "127.6.111.253" port: 39259 } } peers { permanent_uuid: "883f8f327b0e4b72b943de5ef25445f2" member_type: VOTER last_known_addr { host: "127.6.111.252" port: 40087 } }
08:03:39.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:39.120015 7404 leader_election.cc:290] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 98a4571016524600ae1e653f157c03c0 (127.6.111.253:39259), 883f8f327b0e4b72b943de5ef25445f2 (127.6.111.252:40087)
08:03:39.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:39.120398 6722 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "37df082dc4d74885bb14a018ac6c52a1" candidate_term: 4 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "98a4571016524600ae1e653f157c03c0" is_pre_election: true
08:03:39.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:39.120683 6722 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0 [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 37df082dc4d74885bb14a018ac6c52a1 in term 3.
08:03:39.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:39.121173 6620 leader_election.cc:304] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 37df082dc4d74885bb14a018ac6c52a1, 98a4571016524600ae1e653f157c03c0; no voters:
08:03:39.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:39.121471 7404 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [term 3 FOLLOWER]: Leader pre-election won for term 4
08:03:39.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:39.121599 7404 raft_consensus.cc:491] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [term 3 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:03:39.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:39.121697 7404 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [term 3 FOLLOWER]: Advancing to term 4
08:03:39.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:39.121855 6619 leader_election.cc:336] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 883f8f327b0e4b72b943de5ef25445f2 (127.6.111.252:40087): Network error: Client connection negotiation failed: client connection to 127.6.111.252:40087: connect: Connection refused (error 111)
08:03:39.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:39.124521 7404 raft_consensus.cc:513] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "37df082dc4d74885bb14a018ac6c52a1" member_type: VOTER last_known_addr { host: "127.6.111.254" port: 46321 } } peers { permanent_uuid: "98a4571016524600ae1e653f157c03c0" member_type: VOTER last_known_addr { host: "127.6.111.253" port: 39259 } } peers { permanent_uuid: "883f8f327b0e4b72b943de5ef25445f2" member_type: VOTER last_known_addr { host: "127.6.111.252" port: 40087 } }
08:03:39.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:39.125357 6722 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "37df082dc4d74885bb14a018ac6c52a1" candidate_term: 4 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "98a4571016524600ae1e653f157c03c0"
08:03:39.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:39.125530 6722 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0 [term 3 FOLLOWER]: Advancing to term 4
08:03:39.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:39.127750 6722 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0 [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 37df082dc4d74885bb14a018ac6c52a1 in term 4.
08:03:39.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:39.128186 6620 leader_election.cc:304] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 37df082dc4d74885bb14a018ac6c52a1, 98a4571016524600ae1e653f157c03c0; no voters:
08:03:39.128 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:39.129 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.111.252:40087(127.6.111.252:40087)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:39.129 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.111.252:40087: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:39.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:39.129843 7423 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [term 4 FOLLOWER]: Leader election won for term 4
08:03:39.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:39.129891 7404 leader_election.cc:290] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [CANDIDATE]: Term 4 election: Requested vote from peers 98a4571016524600ae1e653f157c03c0 (127.6.111.253:39259), 883f8f327b0e4b72b943de5ef25445f2 (127.6.111.252:40087)
08:03:39.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:39.130229 7423 raft_consensus.cc:695] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [term 4 LEADER]: Becoming Leader. State: Replica: 37df082dc4d74885bb14a018ac6c52a1, State: Running, Role: LEADER
08:03:39.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:39.130849 7423 consensus_queue.cc:237] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "37df082dc4d74885bb14a018ac6c52a1" member_type: VOTER last_known_addr { host: "127.6.111.254" port: 46321 } } peers { permanent_uuid: "98a4571016524600ae1e653f157c03c0" member_type: VOTER last_known_addr { host: "127.6.111.253" port: 39259 } } peers { permanent_uuid: "883f8f327b0e4b72b943de5ef25445f2" member_type: VOTER last_known_addr { host: "127.6.111.252" port: 40087 } }
08:03:39.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:39.131670 6619 leader_election.cc:336] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer 883f8f327b0e4b72b943de5ef25445f2 (127.6.111.252:40087): Network error: Client connection negotiation failed: client connection to 127.6.111.252:40087: connect: Connection refused (error 111)
08:03:39.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:39.132797 7404 sys_catalog.cc:455] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 37df082dc4d74885bb14a018ac6c52a1. Latest consensus state: current_term: 4 leader_uuid: "37df082dc4d74885bb14a018ac6c52a1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "37df082dc4d74885bb14a018ac6c52a1" member_type: VOTER last_known_addr { host: "127.6.111.254" port: 46321 } } peers { permanent_uuid: "98a4571016524600ae1e653f157c03c0" member_type: VOTER last_known_addr { host: "127.6.111.253" port: 39259 } } peers { permanent_uuid: "883f8f327b0e4b72b943de5ef25445f2" member_type: VOTER last_known_addr { host: "127.6.111.252" port: 40087 } } }
08:03:39.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:39.132987 7404 sys_catalog.cc:458] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [sys.catalog]: This master's current role is: LEADER
08:03:39.133 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321; will retry
08:03:39.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:39.133783 7425 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
08:03:39.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:39.136013 7425 catalog_manager.cc:671] Loaded metadata for table kudu_system.kudu_transactions [id=33acefd51ec741f1bb55b2c9623413f3]
08:03:39.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:39.136480 7425 catalog_manager.cc:671] Loaded metadata for table txn_manager_fallback [id=e46ea1d2e29e4e89869fdbc2f3d47cdc]
08:03:39.137 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.6.111.252:40087(127.6.111.252:40087)] connecting to peer
08:03:39.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:39.138108 7425 tablet_loader.cc:96] loaded metadata for tablet 469e5c7686c0457dab3668241a8bd9e0 (table txn_manager_fallback [id=e46ea1d2e29e4e89869fdbc2f3d47cdc])
08:03:39.138 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:39.138 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.111.252:40087(127.6.111.252:40087)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:39.138 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.111.252:40087: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:39.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:39.138659 7425 tablet_loader.cc:96] loaded metadata for tablet 7133300aa73e47f6aae2dd4cc4b7a030 (table txn_manager_fallback [id=e46ea1d2e29e4e89869fdbc2f3d47cdc])
08:03:39.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:39.139086 7425 tablet_loader.cc:96] loaded metadata for tablet e7dad1e7bb4f41d4bf4edb315538bedf (table kudu_system.kudu_transactions [id=33acefd51ec741f1bb55b2c9623413f3])
08:03:39.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:39.139356 7425 catalog_manager.cc:1486] Initializing Kudu cluster ID...
08:03:39.140 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321; will retry
08:03:39.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:39.140365 7425 catalog_manager.cc:1261] Loaded cluster ID: f3216bb837ce46b28605338140db648e
08:03:39.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:39.140472 7425 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
08:03:39.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:39.141635 7425 catalog_manager.cc:1506] Loading token signing keys...
08:03:39.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:39.142375 7425 catalog_manager.cc:5966] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1: Loaded TSK: 0
08:03:39.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:39.143126 7425 catalog_manager.cc:1516] Initializing in-progress tserver states...
08:03:39.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:39.155771 7412 tablet_replica.cc:1304] Aborted: operation has been aborted: cancelling pending write operations
08:03:39.157 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.6.111.252:40087(127.6.111.252:40087)] connecting to peer
08:03:39.158 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:39.159 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.111.252:40087(127.6.111.252:40087)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:39.159 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.111.252:40087: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087
08:03:39.161 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
08:03:39.162 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=5, TimeoutTracker(timeout=250, elapsed=83), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.252:40087, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.253:39259, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.254:46321, [1ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.252:40087: Network error: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [3ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.253:39259: OK, [5ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.254:46321: OK, [5ms] delaying RPC due to: Service unavailable: Master config (127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.252:40087, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.253:39259, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.254:46321, [20ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.252:40087: Network error: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [21ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.253:39259: OK, [21ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.254:46321: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [39ms] refreshing cache from master, [48ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.252:40087, [49ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.253:39259, [49ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.254:46321, [51ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.252:40087: Network error: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [52ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.253:39259: OK, [55ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.254:46321: OK, [55ms] delaying RPC due to: Service unavailable: Master config (127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.252:40087, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.253:39259, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.254:46321, [60ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.252:40087: Network error: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [61ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.253:39259: OK, [61ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.254:46321: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.6.111.252:40087,127.6.111.253:39259,127.6.111.254:46321) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.252:40087, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.253:39259, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.111.254:46321, [81ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.252:40087: Network error: Failed to connect to peer master-127.6.111.252:40087(127.6.111.252:40087): Connection refused: /127.6.111.252:40087, [82ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.254:46321: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.6.111.253:39259: OK, deferred=Deferred@12081295(state=PENDING, result=null, callback=(continuation of Deferred@45063573 after retry RPC after error@633763645) -> (continuation of Deferred@1435367748 after retry RPC after error@397655898) -> (continuation of Deferred@1762581260 after retry RPC after error@2137470607) -> (continuation of Deferred@1167763454 after retry RPC after error@1339660664), errback=(continuation of Deferred@45063573 after retry RPC after error@633763645) -> (continuation of Deferred@1435367748 after retry RPC after error@397655898) -> (continuation of Deferred@1762581260 after retry RPC after error@2137470607) -> (continuation of Deferred@1167763454 after retry RPC after error@1339660664))) after lookup
08:03:39.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:39.219508 6722 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0 [term 4 FOLLOWER]: Refusing update from remote peer 37df082dc4d74885bb14a018ac6c52a1: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 4 index: 12. (index mismatch)
08:03:39.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:39.220510 7404 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "98a4571016524600ae1e653f157c03c0" member_type: VOTER last_known_addr { host: "127.6.111.253" port: 39259 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
08:03:39.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:39.224228 7403 sys_catalog.cc:455] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 37df082dc4d74885bb14a018ac6c52a1. Latest consensus state: current_term: 4 leader_uuid: "37df082dc4d74885bb14a018ac6c52a1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "37df082dc4d74885bb14a018ac6c52a1" member_type: VOTER last_known_addr { host: "127.6.111.254" port: 46321 } } peers { permanent_uuid: "98a4571016524600ae1e653f157c03c0" member_type: VOTER last_known_addr { host: "127.6.111.253" port: 39259 } } peers { permanent_uuid: "883f8f327b0e4b72b943de5ef25445f2" member_type: VOTER last_known_addr { host: "127.6.111.252" port: 40087 } } }
08:03:39.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:39.224560 7403 sys_catalog.cc:458] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0 [sys.catalog]: This master's current role is: FOLLOWER
08:03:39.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:39.227595 7403 sys_catalog.cc:455] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "37df082dc4d74885bb14a018ac6c52a1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "37df082dc4d74885bb14a018ac6c52a1" member_type: VOTER last_known_addr { host: "127.6.111.254" port: 46321 } } peers { permanent_uuid: "98a4571016524600ae1e653f157c03c0" member_type: VOTER last_known_addr { host: "127.6.111.253" port: 39259 } } peers { permanent_uuid: "883f8f327b0e4b72b943de5ef25445f2" member_type: VOTER last_known_addr { host: "127.6.111.252" port: 40087 } } }
08:03:39.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:39.227782 7403 sys_catalog.cc:458] T 00000000000000000000000000000000 P 98a4571016524600ae1e653f157c03c0 [sys.catalog]: This master's current role is: FOLLOWER
08:03:39.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:39.232069 7423 sys_catalog.cc:455] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 4 leader_uuid: "37df082dc4d74885bb14a018ac6c52a1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "37df082dc4d74885bb14a018ac6c52a1" member_type: VOTER last_known_addr { host: "127.6.111.254" port: 46321 } } peers { permanent_uuid: "98a4571016524600ae1e653f157c03c0" member_type: VOTER last_known_addr { host: "127.6.111.253" port: 39259 } } peers { permanent_uuid: "883f8f327b0e4b72b943de5ef25445f2" member_type: VOTER last_known_addr { host: "127.6.111.252" port: 40087 } } }
08:03:39.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:39.232326 7423 sys_catalog.cc:458] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 [sys.catalog]: This master's current role is: LEADER
08:03:39.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:39.234328 6619 consensus_peers.cc:487] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 -> Peer 883f8f327b0e4b72b943de5ef25445f2 (127.6.111.252:40087): Couldn't send request to peer 883f8f327b0e4b72b943de5ef25445f2. Status: Network error: Client connection negotiation failed: client connection to 127.6.111.252:40087: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
08:03:39.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:39.239254 6633 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:51206:
08:03:39.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
08:03:39.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
08:03:39.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:39.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
08:03:39.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
08:03:39.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:03:39.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:39.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:03:39.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:03:39.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:39.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:39.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:39.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:39.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
08:03:39.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
08:03:39.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:03:39.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:39.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:03:39.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:03:39.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:39.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:39.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:39.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:39.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
08:03:39.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
08:03:39.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:03:39.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:39.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:03:39.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:03:39.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:39.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:39.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:39.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:39.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
08:03:39.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
08:03:39.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
08:03:39.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:39.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:03:39.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:03:39.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:39.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:39.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:39.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:39.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
08:03:39.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
08:03:39.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
08:03:39.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
08:03:39.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:39.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
08:03:39.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
08:03:39.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:39.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
08:03:39.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:39.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:39.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:39.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
08:03:39.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:39.240975 6633 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
08:03:39.265 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) terminating keepalive task (txn ID 0) due to exception org.apache.kudu.client.NonRecoverableException: Failed to write to server: 090753c2c8b748e9a1d89d70c8001722 (127.6.111.194:44167): transaction ID 0 is not available for further commits: state: ABORTED user: "slave" start_timestamp: 1750838617 last_transition_timestamp: 1750838619
08:03:39.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:39.688892 6619 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.111.252:40087: connect: Connection refused (error 111) [suppressed 18 similar messages]
08:03:39.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:39.689884 6619 consensus_peers.cc:487] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 -> Peer 883f8f327b0e4b72b943de5ef25445f2 (127.6.111.252:40087): Couldn't send request to peer 883f8f327b0e4b72b943de5ef25445f2. Status: Network error: Client connection negotiation failed: client connection to 127.6.111.252:40087: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
08:03:40.078 [DEBUG - main] (Connection.java:575) [peer master-127.6.111.252:40087(127.6.111.252:40087)] disconnecting while in state TERMINATED
08:03:40.078 [DEBUG - main] (Connection.java:575) [peer master-127.6.111.254:46321(127.6.111.254:46321)] disconnecting while in state READY
08:03:40.078 [DEBUG - main] (Connection.java:575) [peer ed40a832914747859ad22b79871417e6(127.6.111.195:33209)] disconnecting while in state READY
08:03:40.079 [DEBUG - main] (Connection.java:575) [peer master-127.6.111.253:39259(127.6.111.253:39259)] disconnecting while in state READY
08:03:40.079 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.6.111.254:46321(127.6.111.254:46321)] handling channelInactive
08:03:40.079 [DEBUG - kudu-nio-1] (Connection.java:239) [peer ed40a832914747859ad22b79871417e6(127.6.111.195:33209)] handling channelInactive
08:03:40.079 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.111.254:46321(127.6.111.254:46321)] cleaning up while in state READY due to: connection closed
08:03:40.079 [DEBUG - kudu-nio-1] (Connection.java:709) [peer ed40a832914747859ad22b79871417e6(127.6.111.195:33209)] cleaning up while in state READY due to: connection closed
08:03:40.080 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
08:03:40.081 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.6.111.253:39259(127.6.111.253:39259)] handling channelInactive
08:03:40.081 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.111.253:39259(127.6.111.253:39259)] cleaning up while in state READY due to: connection closed
08:03:40.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:40.081977 6591 external_mini_cluster.cc:1661] Attempting to check leaks for /tmp/dist-test-task8V1MUJ/build/asan/bin/kudu pid 6886
08:03:40.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:40.095364 7035 generic_service.cc:196] Checking for leaks (request via RPC)
08:03:40.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:40.190568 6591 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task8V1MUJ/build/asan/bin/kudu with pid 6886
08:03:40.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:40.208634 7063 connection.cc:537] client connection to 127.6.111.193:36349 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
08:03:40.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:40.208928 7063 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
08:03:40.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:40.209213 6591 external_mini_cluster.cc:1661] Attempting to check leaks for /tmp/dist-test-task8V1MUJ/build/asan/bin/kudu pid 7045
08:03:40.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:40.210505 7063 consensus_peers.cc:487] T e7dad1e7bb4f41d4bf4edb315538bedf P 090753c2c8b748e9a1d89d70c8001722 -> Peer f7b24a2ea005466d8b25624cc1a9044f (127.6.111.193:36349): Couldn't send request to peer f7b24a2ea005466d8b25624cc1a9044f. Status: Network error: Client connection negotiation failed: client connection to 127.6.111.193:36349: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
08:03:40.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:40.215675 7176 generic_service.cc:196] Checking for leaks (request via RPC)
08:03:40.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:40.229867 6619 consensus_peers.cc:487] T 00000000000000000000000000000000 P 37df082dc4d74885bb14a018ac6c52a1 -> Peer 883f8f327b0e4b72b943de5ef25445f2 (127.6.111.252:40087): Couldn't send request to peer 883f8f327b0e4b72b943de5ef25445f2. Status: Network error: Client connection negotiation failed: client connection to 127.6.111.252:40087: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
08:03:40.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:40.332068 6591 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task8V1MUJ/build/asan/bin/kudu with pid 7045
08:03:40.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:40.349296 7205 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.111.193:36349: connect: Connection refused (error 111)
08:03:40.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:40.350591 7205 consensus_peers.cc:487] T 7133300aa73e47f6aae2dd4cc4b7a030 P ed40a832914747859ad22b79871417e6 -> Peer f7b24a2ea005466d8b25624cc1a9044f (127.6.111.193:36349): Couldn't send request to peer f7b24a2ea005466d8b25624cc1a9044f. Status: Network error: Client connection negotiation failed: client connection to 127.6.111.193:36349: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
08:03:40.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:40.354018 6591 external_mini_cluster.cc:1661] Attempting to check leaks for /tmp/dist-test-task8V1MUJ/build/asan/bin/kudu pid 7186
08:03:40.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:40.360378 7318 generic_service.cc:196] Checking for leaks (request via RPC)
08:03:40.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:40.455081 6591 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task8V1MUJ/build/asan/bin/kudu with pid 7186
08:03:40.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:40.473006 6591 external_mini_cluster.cc:1661] Attempting to check leaks for /tmp/dist-test-task8V1MUJ/build/asan/bin/kudu pid 6602
08:03:40.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:40.474073 6673 generic_service.cc:196] Checking for leaks (request via RPC)
08:03:40.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:40.585528 6591 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task8V1MUJ/build/asan/bin/kudu with pid 6602
08:03:40.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:03:40.605469 6698 connection.cc:537] server connection from 127.0.0.1:60676 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
08:03:40.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:40.605779 6591 external_mini_cluster.cc:1661] Attempting to check leaks for /tmp/dist-test-task8V1MUJ/build/asan/bin/kudu pid 6682
08:03:40.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:40.606738 6754 generic_service.cc:196] Checking for leaks (request via RPC)
08:03:40.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:03:40.710314 6591 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task8V1MUJ/build/asan/bin/kudu with pid 6682
08:03:40.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-06-25T08:03:40Z chronyd exiting
08:03:40.861 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveSwitchesToOtherTxnManager: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Failed to write to server: 090753c2c8b748e9a1d89d70c8001722 (127.6.111.194:44167): transaction ID 0 is not open: state: ABORTED user: "slave" start_timestamp: 1750838617 last_transition_timestamp: 1750838619
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduTransaction.doCommitTransaction(KuduTransaction.java:491) ~[main/:?]
at org.apache.kudu.client.KuduTransaction.commitWithMode(KuduTransaction.java:537) ~[main/:?]
at org.apache.kudu.client.KuduTransaction.commit(KuduTransaction.java:284) ~[main/:?]
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveSwitchesToOtherTxnManager(TestKuduTransaction.java:1152) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.RpcProxy.dispatchTxnManagerError(RpcProxy.java:439) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:298) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1503) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1366) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1415) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Failed to write to server: 090753c2c8b748e9a1d89d70c8001722 (127.6.111.194:44167): transaction ID 0 is not open: state: ABORTED user: "slave" start_timestamp: 1750838617 last_transition_timestamp: 1750838619
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduTransaction.doCommitTransaction(KuduTransaction.java:491)
at org.apache.kudu.client.KuduTransaction.commitWithMode(KuduTransaction.java:537)
at org.apache.kudu.client.KuduTransaction.commit(KuduTransaction.java:284)
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveSwitchesToOtherTxnManager(TestKuduTransaction.java:1152)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.RpcProxy.dispatchTxnManagerError(RpcProxy.java:439)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:298)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1503)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1366)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1415)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more