09:04:04.808 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
09:04:04.808 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskescuMK/test-tmp
09:04:04.808 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/asan/bin
09:04:04.808 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskescuMK/java/build/dist-test/../../../build/asan/bin/kudu, test, mini_cluster, --serialization=pb]
09:04:04.810 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194"
extra_master_flags: "--txn_manager_enabled"
extra_master_flags: "--raft_heartbeat_interval_ms=100"
extra_tserver_flags: "--txn_keepalive_interval_ms=1000"
extra_tserver_flags: "--txn_staleness_tracker_interval_ms=250"
extra_tserver_flags: "--enable_txn_system_client_init=true"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
09:04:05.135 [DEBUG - main] (MiniKuduCluster.java:195) Response:
09:04:05.135 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
09:04:05.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:05.138330 26034 mini_oidc.cc:150] Starting JWKS server
09:04:05.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:05.140796 26034 webserver.cc:492] Webserver started at http://127.0.0.1:45127/ using document root <none> and password file <none>
09:04:05.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:05.141029 26034 mini_oidc.cc:186] Starting OIDC Discovery server
09:04:05.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:05.141549 26034 webserver.cc:492] Webserver started at http://127.0.0.1:43727/ using document root <none> and password file <none>
09:04:05.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-29T09:04:05Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
09:04:05.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-29T09:04:05Z Disabled control of system clock
09:04:05.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:05.172466 26034 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskescuMK/build/asan/bin/kudu
09:04:05.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskescuMK/build/asan/bin/kudu
09:04:05.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/master-0/wal
09:04:05.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/master-0/data
09:04:05.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:04:05.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:04:05.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:04:05.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:04:05.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:04:05.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:04:05.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/master-0/logs
09:04:05.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/master-0/data/info.pb
09:04:05.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:04:05.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:04:05.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:04:05.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:04:05.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:04:05.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:04:05.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:04:05.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:04:05.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:04:05.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:04:05.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:04:05.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:04:05.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.108.190:40189
09:04:05.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.108.190
09:04:05.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:04:05.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.108.148:42419
09:04:05.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:04:05.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:04:05.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:04:05.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:04:05.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.108.190:40189,127.25.108.189:37911,127.25.108.188:38177
09:04:05.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:04:05.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45127/jwks.json
09:04:05.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
09:04:05.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
09:04:05.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:05.496769 26045 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:04:05.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:05.497080 26045 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:04:05.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:05.497162 26045 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:04:05.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:05.506292 26045 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
09:04:05.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:05.506614 26045 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
09:04:05.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:05.506809 26045 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:04:05.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:05.506892 26045 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
09:04:05.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:05.506974 26045 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
09:04:05.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:05.507045 26045 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
09:04:05.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:05.507122 26045 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45127/jwks.json
09:04:05.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:05.517861 26045 master_runner.cc:386] Master server non-default flags:
09:04:05.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:04:05.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.108.148:42419
09:04:05.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:04:05.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:04:05.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
09:04:05.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/master-0/data
09:04:05.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/master-0/wal
09:04:05.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:04:05.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.108.190:40189,127.25.108.189:37911,127.25.108.188:38177
09:04:05.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
09:04:05.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:04:05.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:04:05.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:04:05.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.108.190:40189
09:04:05.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:04:05.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:04:05.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:04:05.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45127/jwks.json
09:04:05.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:04:05.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:04:05.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/master-0/data/info.pb
09:04:05.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.108.190
09:04:05.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:04:05.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:04:05.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:04:05.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:04:05.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:04:05.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:04:05.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/master-0/logs
09:04:05.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:04:05.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:04:05.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:04:05.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:04:05.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:04:05.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 60f5e5267b92c39485a66121d3ce3cc7ef57b0e0
09:04:05.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:04:05.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 29 Oct 2025 08:43:21 UTC on 24a791456cd2
09:04:05.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8772
09:04:05.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
09:04:05.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:05.520366 26045 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:04:05.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:05.522114 26045 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:04:05.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:05.530661 26050 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:04:05.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:05.532506 26051 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:04:05.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:05.532645 26053 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:04:05.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:05.533841 26045 server_base.cc:1047] running on GCE node
09:04:05.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:05.534881 26045 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:04:05.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:05.536525 26045 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:04:05.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:05.537779 26045 hybrid_clock.cc:648] HybridClock initialized: now 1761728645537705 us; error 43 us; skew 500 ppm
09:04:05.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:05.538129 26045 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:04:05.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:05.540376 26045 webserver.cc:492] Webserver started at http://127.25.108.190:39581/ using document root <none> and password file <none>
09:04:05.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:05.541110 26045 fs_manager.cc:362] Metadata directory not provided
09:04:05.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:05.541244 26045 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:04:05.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:05.541610 26045 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
09:04:05.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:05.544027 26045 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/master-0/data/instance:
09:04:05.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aaeb9a61bdca4ab8bdc0ee318551ab52"
09:04:05.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 09:04:05 on dist-test-slave-rph9"
09:04:05.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:05.544854 26045 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/master-0/wal/instance:
09:04:05.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aaeb9a61bdca4ab8bdc0ee318551ab52"
09:04:05.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 09:04:05 on dist-test-slave-rph9"
09:04:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:05.550261 26045 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.005s sys 0.000s
09:04:05.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:05.553547 26059 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:04:05.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:05.554867 26045 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.001s sys 0.000s
09:04:05.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:05.555048 26045 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/master-0/data,/tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/master-0/wal
09:04:05.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aaeb9a61bdca4ab8bdc0ee318551ab52"
09:04:05.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 09:04:05 on dist-test-slave-rph9"
09:04:05.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:05.555240 26045 fs_report.cc:389] FS layout report
09:04:05.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:04:05.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/master-0/wal
09:04:05.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/master-0/wal
09:04:05.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/master-0/data/data
09:04:05.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:04:05.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:04:05.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:04:05.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:04:05.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:04:05.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:04:05.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:04:05.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:04:05.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:04:05.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:04:05.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:04:05.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:05.589533 26045 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:04:05.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:05.590461 26045 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:04:05.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:05.590775 26045 kserver.cc:163] Server-wide thread pool size limit: 3276
09:04:05.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:05.612944 26045 rpc_server.cc:307] RPC server started. Bound to: 127.25.108.190:40189
09:04:05.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:05.612982 26120 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.25.108.190:40189 every 8 connection(s)
09:04:05.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:05.614454 26045 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/master-0/data/info.pb
09:04:05.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:05.618551 26121 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
09:04:05.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:05.623319 26034 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskescuMK/build/asan/bin/kudu as pid 26045
09:04:05.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:05.623618 26034 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/master-0/wal/instance
09:04:05.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:05.624293 26121 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.108.190" port: 40189 } has no permanent_uuid. Determining permanent_uuid...
09:04:05.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:05.625936 26034 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskescuMK/build/asan/bin/kudu
09:04:05.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskescuMK/build/asan/bin/kudu
09:04:05.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/master-1/wal
09:04:05.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/master-1/data
09:04:05.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:04:05.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:04:05.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:04:05.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:04:05.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:04:05.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:04:05.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/master-1/logs
09:04:05.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/master-1/data/info.pb
09:04:05.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:04:05.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:04:05.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:04:05.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:04:05.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:04:05.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:04:05.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:04:05.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:04:05.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:04:05.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:04:05.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:04:05.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:04:05.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.108.189:37911
09:04:05.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.108.189
09:04:05.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:04:05.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.108.148:42419
09:04:05.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:04:05.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:04:05.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:04:05.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:04:05.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.108.190:40189,127.25.108.189:37911,127.25.108.188:38177
09:04:05.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:04:05.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45127/jwks.json
09:04:05.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
09:04:05.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
09:04:05.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:05.640359 26121 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.108.189" port: 37911 } has no permanent_uuid. Determining permanent_uuid...
09:04:05.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:05.641507 26061 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.25.108.189:37911: connect: Connection refused (error 111)
09:04:05.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:05.643121 26121 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.25.108.189:37911: Network error: Client connection negotiation failed: client connection to 127.25.108.189:37911: connect: Connection refused (error 111)
09:04:05.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:05.692564 26121 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.108.189" port: 37911 } attempt: 1
09:04:05.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:05.694917 26121 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.25.108.189:37911: Network error: Client connection negotiation failed: client connection to 127.25.108.189:37911: connect: Connection refused (error 111)
09:04:05.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:05.768083 26121 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.108.189" port: 37911 } attempt: 2
09:04:05.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:05.770191 26121 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.25.108.189:37911: Network error: Client connection negotiation failed: client connection to 127.25.108.189:37911: connect: Connection refused (error 111)
09:04:05.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:05.861475 26121 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.108.189" port: 37911 } attempt: 3
09:04:05.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:05.863484 26121 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.25.108.189:37911: Network error: Client connection negotiation failed: client connection to 127.25.108.189:37911: connect: Connection refused (error 111)
09:04:05.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:05.964066 26125 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:04:05.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:05.964401 26125 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:04:05.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:05.964480 26125 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:04:05.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:05.972657 26125 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
09:04:05.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:05.972780 26125 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
09:04:05.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:05.972828 26125 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:04:05.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:05.972867 26125 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
09:04:05.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:05.972920 26125 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
09:04:05.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:05.972959 26125 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
09:04:05.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:05.973019 26125 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45127/jwks.json
09:04:05.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:05.983093 26125 master_runner.cc:386] Master server non-default flags:
09:04:05.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:04:05.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.108.148:42419
09:04:05.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:04:05.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:04:05.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
09:04:05.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/master-1/data
09:04:05.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/master-1/wal
09:04:05.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:04:05.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.108.190:40189,127.25.108.189:37911,127.25.108.188:38177
09:04:05.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
09:04:05.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:04:05.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:04:05.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:04:05.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.108.189:37911
09:04:05.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:04:05.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:04:05.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:04:05.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45127/jwks.json
09:04:05.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:04:05.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:04:05.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/master-1/data/info.pb
09:04:05.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.108.189
09:04:05.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:04:05.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:04:05.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:04:05.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:04:05.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:04:05.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:04:05.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/master-1/logs
09:04:05.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:04:05.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:04:05.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:04:05.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:04:05.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:04:05.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 60f5e5267b92c39485a66121d3ce3cc7ef57b0e0
09:04:05.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:04:05.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 29 Oct 2025 08:43:21 UTC on 24a791456cd2
09:04:05.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8772
09:04:05.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
09:04:05.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:05.984879 26125 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:04:05.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:05.986636 26125 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:04:05.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:05.994319 26131 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:04:05.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:05.994369 26132 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:04:05.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:05.995277 26134 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:04:05.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:05.995645 26125 server_base.cc:1047] running on GCE node
09:04:05.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:05.996421 26125 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:04:05.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:05.997859 26125 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:04:05.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:05.999075 26125 hybrid_clock.cc:648] HybridClock initialized: now 1761728645999008 us; error 44 us; skew 500 ppm
09:04:05.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:05.999372 26125 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:04:06.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.001585 26125 webserver.cc:492] Webserver started at http://127.25.108.189:45135/ using document root <none> and password file <none>
09:04:06.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.002293 26125 fs_manager.cc:362] Metadata directory not provided
09:04:06.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.002398 26125 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:04:06.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.002719 26125 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
09:04:06.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.005275 26125 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/master-1/data/instance:
09:04:06.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0ec5572f7e8c4d27bb850d5983c8eb07"
09:04:06.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 09:04:05 on dist-test-slave-rph9"
09:04:06.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.006007 26125 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/master-1/wal/instance:
09:04:06.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0ec5572f7e8c4d27bb850d5983c8eb07"
09:04:06.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 09:04:05 on dist-test-slave-rph9"
09:04:06.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.006762 26121 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.108.189" port: 37911 } attempt: 4
09:04:06.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:06.008760 26121 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.25.108.189:37911: Network error: Client connection negotiation failed: client connection to 127.25.108.189:37911: connect: Connection refused (error 111)
09:04:06.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.011868 26125 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.004s sys 0.003s
09:04:06.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.015478 26140 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:04:06.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.017056 26125 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
09:04:06.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.017210 26125 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/master-1/data,/tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/master-1/wal
09:04:06.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0ec5572f7e8c4d27bb850d5983c8eb07"
09:04:06.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 09:04:05 on dist-test-slave-rph9"
09:04:06.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.017377 26125 fs_report.cc:389] FS layout report
09:04:06.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:04:06.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/master-1/wal
09:04:06.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/master-1/wal
09:04:06.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/master-1/data/data
09:04:06.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:04:06.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:04:06.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:04:06.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:04:06.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:04:06.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:04:06.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:04:06.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:04:06.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:04:06.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:04:06.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:04:06.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.030580 26125 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:04:06.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.031368 26125 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:04:06.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.031612 26125 kserver.cc:163] Server-wide thread pool size limit: 3276
09:04:06.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.054463 26125 rpc_server.cc:307] RPC server started. Bound to: 127.25.108.189:37911
09:04:06.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.054481 26201 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.25.108.189:37911 every 8 connection(s)
09:04:06.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.055989 26125 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/master-1/data/info.pb
09:04:06.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.060148 26202 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
09:04:06.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.064543 26034 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskescuMK/build/asan/bin/kudu as pid 26125
09:04:06.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.064673 26034 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/master-1/wal/instance
09:04:06.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.065584 26034 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskescuMK/build/asan/bin/kudu
09:04:06.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskescuMK/build/asan/bin/kudu
09:04:06.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/master-2/wal
09:04:06.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/master-2/data
09:04:06.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:04:06.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:04:06.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:04:06.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:04:06.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:04:06.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:04:06.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/master-2/logs
09:04:06.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/master-2/data/info.pb
09:04:06.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:04:06.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:04:06.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:04:06.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:04:06.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:04:06.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:04:06.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:04:06.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:04:06.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:04:06.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:04:06.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:04:06.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:04:06.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.108.188:38177
09:04:06.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.108.188
09:04:06.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:04:06.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.108.148:42419
09:04:06.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:04:06.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:04:06.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:04:06.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:04:06.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.108.190:40189,127.25.108.189:37911,127.25.108.188:38177
09:04:06.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:04:06.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45127/jwks.json
09:04:06.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
09:04:06.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
09:04:06.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.066000 26202 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.108.190" port: 40189 } has no permanent_uuid. Determining permanent_uuid...
09:04:06.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.084806 26202 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.108.189" port: 37911 } has no permanent_uuid. Determining permanent_uuid...
09:04:06.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.095017 26202 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.108.188" port: 38177 } has no permanent_uuid. Determining permanent_uuid...
09:04:06.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:06.096225 26143 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.25.108.188:38177: connect: Connection refused (error 111)
09:04:06.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:06.097843 26202 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.25.108.188:38177: Network error: Client connection negotiation failed: client connection to 127.25.108.188:38177: connect: Connection refused (error 111)
09:04:06.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.147159 26202 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.108.188" port: 38177 } attempt: 1
09:04:06.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:06.149065 26202 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.25.108.188:38177: Network error: Client connection negotiation failed: client connection to 127.25.108.188:38177: connect: Connection refused (error 111)
09:04:06.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.217362 26202 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.108.188" port: 38177 } attempt: 2
09:04:06.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:06.219368 26202 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.25.108.188:38177: Network error: Client connection negotiation failed: client connection to 127.25.108.188:38177: connect: Connection refused (error 111)
09:04:06.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.308040 26121 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.108.189" port: 37911 } attempt: 5
09:04:06.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.310675 26202 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.108.188" port: 38177 } attempt: 3
09:04:06.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:06.312866 26202 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.25.108.188:38177: Network error: Client connection negotiation failed: client connection to 127.25.108.188:38177: connect: Connection refused (error 111)
09:04:06.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.314694 26121 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.108.188" port: 38177 } has no permanent_uuid. Determining permanent_uuid...
09:04:06.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:06.316390 26121 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.25.108.188:38177: Network error: Client connection negotiation failed: client connection to 127.25.108.188:38177: connect: Connection refused (error 111)
09:04:06.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.367614 26121 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.108.188" port: 38177 } attempt: 1
09:04:06.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:06.369747 26121 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.25.108.188:38177: Network error: Client connection negotiation failed: client connection to 127.25.108.188:38177: connect: Connection refused (error 111)
09:04:06.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:06.400555 26204 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:04:06.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:06.400862 26204 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:04:06.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:06.400934 26204 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:04:06.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:06.408921 26204 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
09:04:06.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:06.409034 26204 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
09:04:06.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:06.409082 26204 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:04:06.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:06.409122 26204 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
09:04:06.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:06.409174 26204 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
09:04:06.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:06.409214 26204 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
09:04:06.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:06.409268 26204 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45127/jwks.json
09:04:06.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.419188 26204 master_runner.cc:386] Master server non-default flags:
09:04:06.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:04:06.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.108.148:42419
09:04:06.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:04:06.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:04:06.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
09:04:06.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/master-2/data
09:04:06.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/master-2/wal
09:04:06.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:04:06.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.108.190:40189,127.25.108.189:37911,127.25.108.188:38177
09:04:06.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
09:04:06.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:04:06.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:04:06.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:04:06.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.108.188:38177
09:04:06.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:04:06.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:04:06.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:04:06.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45127/jwks.json
09:04:06.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:04:06.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:04:06.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/master-2/data/info.pb
09:04:06.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.108.188
09:04:06.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:04:06.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:04:06.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:04:06.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:04:06.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:04:06.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:04:06.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/master-2/logs
09:04:06.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:04:06.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:04:06.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:04:06.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:04:06.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:04:06.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 60f5e5267b92c39485a66121d3ce3cc7ef57b0e0
09:04:06.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:04:06.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 29 Oct 2025 08:43:21 UTC on 24a791456cd2
09:04:06.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8772
09:04:06.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
09:04:06.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.420951 26204 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:04:06.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.422623 26204 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:04:06.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:06.430502 26212 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:04:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:06.430505 26213 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:04:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:06.431308 26215 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:04:06.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.432201 26204 server_base.cc:1047] running on GCE node
09:04:06.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.432919 26204 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:04:06.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.434244 26204 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:04:06.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.435454 26204 hybrid_clock.cc:648] HybridClock initialized: now 1761728646435392 us; error 39 us; skew 500 ppm
09:04:06.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.435777 26204 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:04:06.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.437875 26204 webserver.cc:492] Webserver started at http://127.25.108.188:40567/ using document root <none> and password file <none>
09:04:06.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.437999 26121 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.108.188" port: 38177 } attempt: 2
09:04:06.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.438619 26204 fs_manager.cc:362] Metadata directory not provided
09:04:06.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.438787 26204 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:04:06.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.439199 26204 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
09:04:06.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:06.439849 26121 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.25.108.188:38177: Network error: Client connection negotiation failed: client connection to 127.25.108.188:38177: connect: Connection refused (error 111)
09:04:06.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.441671 26204 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/master-2/data/instance:
09:04:06.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "db0c22fe226d4071b623e4f6921f4a3d"
09:04:06.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 09:04:06 on dist-test-slave-rph9"
09:04:06.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.442399 26204 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/master-2/wal/instance:
09:04:06.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "db0c22fe226d4071b623e4f6921f4a3d"
09:04:06.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 09:04:06 on dist-test-slave-rph9"
09:04:06.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.447481 26204 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.003s sys 0.003s
09:04:06.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.451112 26221 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:04:06.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.452587 26204 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
09:04:06.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.452764 26204 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/master-2/data,/tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/master-2/wal
09:04:06.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "db0c22fe226d4071b623e4f6921f4a3d"
09:04:06.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 09:04:06 on dist-test-slave-rph9"
09:04:06.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.452997 26204 fs_report.cc:389] FS layout report
09:04:06.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:04:06.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/master-2/wal
09:04:06.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/master-2/wal
09:04:06.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/master-2/data/data
09:04:06.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:04:06.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:04:06.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:04:06.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:04:06.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:04:06.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:04:06.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:04:06.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:04:06.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:04:06.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:04:06.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:04:06.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.456091 26202 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.108.188" port: 38177 } attempt: 4
09:04:06.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:06.457890 26202 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.25.108.188:38177: Network error: Client connection negotiation failed: client connection to 127.25.108.188:38177: connect: Connection refused (error 111)
09:04:06.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.473663 26204 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:04:06.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.474565 26204 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:04:06.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.474881 26204 kserver.cc:163] Server-wide thread pool size limit: 3276
09:04:06.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.497339 26204 rpc_server.cc:307] RPC server started. Bound to: 127.25.108.188:38177
09:04:06.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.497381 26282 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.25.108.188:38177 every 8 connection(s)
09:04:06.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.498943 26204 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/master-2/data/info.pb
09:04:06.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.503094 26283 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
09:04:06.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.504598 26034 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskescuMK/build/asan/bin/kudu as pid 26204
09:04:06.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.504804 26034 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/master-2/wal/instance
09:04:06.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.510450 26283 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.108.190" port: 40189 } has no permanent_uuid. Determining permanent_uuid...
09:04:06.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.536535 26283 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.108.189" port: 37911 } has no permanent_uuid. Determining permanent_uuid...
09:04:06.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.543023 26283 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.108.188" port: 38177 } has no permanent_uuid. Determining permanent_uuid...
09:04:06.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.546096 26121 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.108.188" port: 38177 } attempt: 3
09:04:06.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.554059 26283 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P db0c22fe226d4071b623e4f6921f4a3d: Bootstrap starting.
09:04:06.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.557528 26121 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P aaeb9a61bdca4ab8bdc0ee318551ab52: Bootstrap starting.
09:04:06.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.558010 26283 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P db0c22fe226d4071b623e4f6921f4a3d: Neither blocks nor log segments found. Creating new log.
09:04:06.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.559403 26283 log.cc:826] T 00000000000000000000000000000000 P db0c22fe226d4071b623e4f6921f4a3d: Log is configured to *not* fsync() on all Append() calls
09:04:06.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.561515 26121 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P aaeb9a61bdca4ab8bdc0ee318551ab52: Neither blocks nor log segments found. Creating new log.
09:04:06.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.562429 26283 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P db0c22fe226d4071b623e4f6921f4a3d: No bootstrap required, opened a new log
09:04:06.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.562808 26121 log.cc:826] T 00000000000000000000000000000000 P aaeb9a61bdca4ab8bdc0ee318551ab52: Log is configured to *not* fsync() on all Append() calls
09:04:06.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.565514 26121 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P aaeb9a61bdca4ab8bdc0ee318551ab52: No bootstrap required, opened a new log
09:04:06.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.569697 26283 raft_consensus.cc:359] T 00000000000000000000000000000000 P db0c22fe226d4071b623e4f6921f4a3d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aaeb9a61bdca4ab8bdc0ee318551ab52" member_type: VOTER last_known_addr { host: "127.25.108.190" port: 40189 } } peers { permanent_uuid: "0ec5572f7e8c4d27bb850d5983c8eb07" member_type: VOTER last_known_addr { host: "127.25.108.189" port: 37911 } } peers { permanent_uuid: "db0c22fe226d4071b623e4f6921f4a3d" member_type: VOTER last_known_addr { host: "127.25.108.188" port: 38177 } }
09:04:06.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.570073 26283 raft_consensus.cc:385] T 00000000000000000000000000000000 P db0c22fe226d4071b623e4f6921f4a3d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:04:06.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.570185 26283 raft_consensus.cc:740] T 00000000000000000000000000000000 P db0c22fe226d4071b623e4f6921f4a3d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: db0c22fe226d4071b623e4f6921f4a3d, State: Initialized, Role: FOLLOWER
09:04:06.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.570873 26283 consensus_queue.cc:260] T 00000000000000000000000000000000 P db0c22fe226d4071b623e4f6921f4a3d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aaeb9a61bdca4ab8bdc0ee318551ab52" member_type: VOTER last_known_addr { host: "127.25.108.190" port: 40189 } } peers { permanent_uuid: "0ec5572f7e8c4d27bb850d5983c8eb07" member_type: VOTER last_known_addr { host: "127.25.108.189" port: 37911 } } peers { permanent_uuid: "db0c22fe226d4071b623e4f6921f4a3d" member_type: VOTER last_known_addr { host: "127.25.108.188" port: 38177 } }
09:04:06.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.571763 26292 sys_catalog.cc:455] T 00000000000000000000000000000000 P db0c22fe226d4071b623e4f6921f4a3d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aaeb9a61bdca4ab8bdc0ee318551ab52" member_type: VOTER last_known_addr { host: "127.25.108.190" port: 40189 } } peers { permanent_uuid: "0ec5572f7e8c4d27bb850d5983c8eb07" member_type: VOTER last_known_addr { host: "127.25.108.189" port: 37911 } } peers { permanent_uuid: "db0c22fe226d4071b623e4f6921f4a3d" member_type: VOTER last_known_addr { host: "127.25.108.188" port: 38177 } } }
09:04:06.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.572093 26292 sys_catalog.cc:458] T 00000000000000000000000000000000 P db0c22fe226d4071b623e4f6921f4a3d [sys.catalog]: This master's current role is: FOLLOWER
09:04:06.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.572767 26121 raft_consensus.cc:359] T 00000000000000000000000000000000 P aaeb9a61bdca4ab8bdc0ee318551ab52 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aaeb9a61bdca4ab8bdc0ee318551ab52" member_type: VOTER last_known_addr { host: "127.25.108.190" port: 40189 } } peers { permanent_uuid: "0ec5572f7e8c4d27bb850d5983c8eb07" member_type: VOTER last_known_addr { host: "127.25.108.189" port: 37911 } } peers { permanent_uuid: "db0c22fe226d4071b623e4f6921f4a3d" member_type: VOTER last_known_addr { host: "127.25.108.188" port: 38177 } }
09:04:06.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.573072 26283 sys_catalog.cc:565] T 00000000000000000000000000000000 P db0c22fe226d4071b623e4f6921f4a3d [sys.catalog]: configured and running, proceeding with master startup.
09:04:06.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.573072 26121 raft_consensus.cc:385] T 00000000000000000000000000000000 P aaeb9a61bdca4ab8bdc0ee318551ab52 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:04:06.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.573191 26121 raft_consensus.cc:740] T 00000000000000000000000000000000 P aaeb9a61bdca4ab8bdc0ee318551ab52 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: aaeb9a61bdca4ab8bdc0ee318551ab52, State: Initialized, Role: FOLLOWER
09:04:06.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.573787 26121 consensus_queue.cc:260] T 00000000000000000000000000000000 P aaeb9a61bdca4ab8bdc0ee318551ab52 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aaeb9a61bdca4ab8bdc0ee318551ab52" member_type: VOTER last_known_addr { host: "127.25.108.190" port: 40189 } } peers { permanent_uuid: "0ec5572f7e8c4d27bb850d5983c8eb07" member_type: VOTER last_known_addr { host: "127.25.108.189" port: 37911 } } peers { permanent_uuid: "db0c22fe226d4071b623e4f6921f4a3d" member_type: VOTER last_known_addr { host: "127.25.108.188" port: 38177 } }
09:04:06.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.575623 26293 sys_catalog.cc:455] T 00000000000000000000000000000000 P aaeb9a61bdca4ab8bdc0ee318551ab52 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aaeb9a61bdca4ab8bdc0ee318551ab52" member_type: VOTER last_known_addr { host: "127.25.108.190" port: 40189 } } peers { permanent_uuid: "0ec5572f7e8c4d27bb850d5983c8eb07" member_type: VOTER last_known_addr { host: "127.25.108.189" port: 37911 } } peers { permanent_uuid: "db0c22fe226d4071b623e4f6921f4a3d" member_type: VOTER last_known_addr { host: "127.25.108.188" port: 38177 } } }
09:04:06.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.576107 26293 sys_catalog.cc:458] T 00000000000000000000000000000000 P aaeb9a61bdca4ab8bdc0ee318551ab52 [sys.catalog]: This master's current role is: FOLLOWER
09:04:06.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.577342 26121 sys_catalog.cc:565] T 00000000000000000000000000000000 P aaeb9a61bdca4ab8bdc0ee318551ab52 [sys.catalog]: configured and running, proceeding with master startup.
09:04:06.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:06.586431 26306 catalog_manager.cc:1568] T 00000000000000000000000000000000 P db0c22fe226d4071b623e4f6921f4a3d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
09:04:06.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:06.586561 26306 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
09:04:06.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.582690 26204 master_runner.cc:186] Error getting master registration for 127.25.108.190:40189: OK, instance_id { permanent_uuid: "aaeb9a61bdca4ab8bdc0ee318551ab52" instance_seqno: 1761728645608205 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
09:04:06.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.591861 26204 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
09:04:06.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:06.593726 26065 tablet.cc:2378] T 00000000000000000000000000000000 P aaeb9a61bdca4ab8bdc0ee318551ab52: Can't schedule compaction. Clean time has not been advanced past its initial value.
09:04:06.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:06.599946 26316 catalog_manager.cc:1568] T 00000000000000000000000000000000 P aaeb9a61bdca4ab8bdc0ee318551ab52: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
09:04:06.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:06.600091 26316 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
09:04:06.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.599323 26045 master_runner.cc:186] Error getting master registration for 127.25.108.189:37911: OK, instance_id { permanent_uuid: "0ec5572f7e8c4d27bb850d5983c8eb07" instance_seqno: 1761728646048411 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
09:04:06.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.603626 26045 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
09:04:06.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.621675 26293 raft_consensus.cc:493] T 00000000000000000000000000000000 P aaeb9a61bdca4ab8bdc0ee318551ab52 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
09:04:06.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.621981 26293 raft_consensus.cc:515] T 00000000000000000000000000000000 P aaeb9a61bdca4ab8bdc0ee318551ab52 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aaeb9a61bdca4ab8bdc0ee318551ab52" member_type: VOTER last_known_addr { host: "127.25.108.190" port: 40189 } } peers { permanent_uuid: "0ec5572f7e8c4d27bb850d5983c8eb07" member_type: VOTER last_known_addr { host: "127.25.108.189" port: 37911 } } peers { permanent_uuid: "db0c22fe226d4071b623e4f6921f4a3d" member_type: VOTER last_known_addr { host: "127.25.108.188" port: 38177 } }
09:04:06.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.624969 26248 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "aaeb9a61bdca4ab8bdc0ee318551ab52" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "db0c22fe226d4071b623e4f6921f4a3d" is_pre_election: true
09:04:06.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.625541 26248 raft_consensus.cc:2468] T 00000000000000000000000000000000 P db0c22fe226d4071b623e4f6921f4a3d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate aaeb9a61bdca4ab8bdc0ee318551ab52 in term 0.
09:04:06.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.626027 26293 leader_election.cc:290] T 00000000000000000000000000000000 P aaeb9a61bdca4ab8bdc0ee318551ab52 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 0ec5572f7e8c4d27bb850d5983c8eb07 (127.25.108.189:37911), db0c22fe226d4071b623e4f6921f4a3d (127.25.108.188:38177)
09:04:06.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.626693 26062 leader_election.cc:304] T 00000000000000000000000000000000 P aaeb9a61bdca4ab8bdc0ee318551ab52 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: aaeb9a61bdca4ab8bdc0ee318551ab52, db0c22fe226d4071b623e4f6921f4a3d; no voters:
09:04:06.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.627154 26293 raft_consensus.cc:2804] T 00000000000000000000000000000000 P aaeb9a61bdca4ab8bdc0ee318551ab52 [term 0 FOLLOWER]: Leader pre-election won for term 1
09:04:06.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.627329 26293 raft_consensus.cc:493] T 00000000000000000000000000000000 P aaeb9a61bdca4ab8bdc0ee318551ab52 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
09:04:06.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.627451 26293 raft_consensus.cc:3060] T 00000000000000000000000000000000 P aaeb9a61bdca4ab8bdc0ee318551ab52 [term 0 FOLLOWER]: Advancing to term 1
09:04:06.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.633625 26293 raft_consensus.cc:515] T 00000000000000000000000000000000 P aaeb9a61bdca4ab8bdc0ee318551ab52 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aaeb9a61bdca4ab8bdc0ee318551ab52" member_type: VOTER last_known_addr { host: "127.25.108.190" port: 40189 } } peers { permanent_uuid: "0ec5572f7e8c4d27bb850d5983c8eb07" member_type: VOTER last_known_addr { host: "127.25.108.189" port: 37911 } } peers { permanent_uuid: "db0c22fe226d4071b623e4f6921f4a3d" member_type: VOTER last_known_addr { host: "127.25.108.188" port: 38177 } }
09:04:06.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.635387 26293 leader_election.cc:290] T 00000000000000000000000000000000 P aaeb9a61bdca4ab8bdc0ee318551ab52 [CANDIDATE]: Term 1 election: Requested vote from peers 0ec5572f7e8c4d27bb850d5983c8eb07 (127.25.108.189:37911), db0c22fe226d4071b623e4f6921f4a3d (127.25.108.188:38177)
09:04:06.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.635492 26248 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "aaeb9a61bdca4ab8bdc0ee318551ab52" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "db0c22fe226d4071b623e4f6921f4a3d"
09:04:06.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.635730 26248 raft_consensus.cc:3060] T 00000000000000000000000000000000 P db0c22fe226d4071b623e4f6921f4a3d [term 0 FOLLOWER]: Advancing to term 1
09:04:06.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.639307 26248 raft_consensus.cc:2468] T 00000000000000000000000000000000 P db0c22fe226d4071b623e4f6921f4a3d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate aaeb9a61bdca4ab8bdc0ee318551ab52 in term 1.
09:04:06.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.639756 26062 leader_election.cc:304] T 00000000000000000000000000000000 P aaeb9a61bdca4ab8bdc0ee318551ab52 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: aaeb9a61bdca4ab8bdc0ee318551ab52, db0c22fe226d4071b623e4f6921f4a3d; no voters:
09:04:06.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.641837 26293 raft_consensus.cc:2804] T 00000000000000000000000000000000 P aaeb9a61bdca4ab8bdc0ee318551ab52 [term 1 FOLLOWER]: Leader election won for term 1
09:04:06.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.642668 26293 raft_consensus.cc:697] T 00000000000000000000000000000000 P aaeb9a61bdca4ab8bdc0ee318551ab52 [term 1 LEADER]: Becoming Leader. State: Replica: aaeb9a61bdca4ab8bdc0ee318551ab52, State: Running, Role: LEADER
09:04:06.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.643388 26293 consensus_queue.cc:237] T 00000000000000000000000000000000 P aaeb9a61bdca4ab8bdc0ee318551ab52 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aaeb9a61bdca4ab8bdc0ee318551ab52" member_type: VOTER last_known_addr { host: "127.25.108.190" port: 40189 } } peers { permanent_uuid: "0ec5572f7e8c4d27bb850d5983c8eb07" member_type: VOTER last_known_addr { host: "127.25.108.189" port: 37911 } } peers { permanent_uuid: "db0c22fe226d4071b623e4f6921f4a3d" member_type: VOTER last_known_addr { host: "127.25.108.188" port: 38177 } }
09:04:06.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.658599 26318 sys_catalog.cc:455] T 00000000000000000000000000000000 P aaeb9a61bdca4ab8bdc0ee318551ab52 [sys.catalog]: SysCatalogTable state changed. Reason: New leader aaeb9a61bdca4ab8bdc0ee318551ab52. Latest consensus state: current_term: 1 leader_uuid: "aaeb9a61bdca4ab8bdc0ee318551ab52" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aaeb9a61bdca4ab8bdc0ee318551ab52" member_type: VOTER last_known_addr { host: "127.25.108.190" port: 40189 } } peers { permanent_uuid: "0ec5572f7e8c4d27bb850d5983c8eb07" member_type: VOTER last_known_addr { host: "127.25.108.189" port: 37911 } } peers { permanent_uuid: "db0c22fe226d4071b623e4f6921f4a3d" member_type: VOTER last_known_addr { host: "127.25.108.188" port: 38177 } } }
09:04:06.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.659386 26318 sys_catalog.cc:458] T 00000000000000000000000000000000 P aaeb9a61bdca4ab8bdc0ee318551ab52 [sys.catalog]: This master's current role is: LEADER
09:04:06.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.660706 26320 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
09:04:06.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.663101 26320 catalog_manager.cc:1494] Initializing Kudu cluster ID...
09:04:06.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.674937 26248 raft_consensus.cc:1275] T 00000000000000000000000000000000 P db0c22fe226d4071b623e4f6921f4a3d [term 1 FOLLOWER]: Refusing update from remote peer aaeb9a61bdca4ab8bdc0ee318551ab52: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
09:04:06.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.675973 26293 consensus_queue.cc:1048] T 00000000000000000000000000000000 P aaeb9a61bdca4ab8bdc0ee318551ab52 [LEADER]: Connected to new peer: Peer: permanent_uuid: "db0c22fe226d4071b623e4f6921f4a3d" member_type: VOTER last_known_addr { host: "127.25.108.188" port: 38177 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
09:04:06.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.694164 26292 sys_catalog.cc:455] T 00000000000000000000000000000000 P db0c22fe226d4071b623e4f6921f4a3d [sys.catalog]: SysCatalogTable state changed. Reason: New leader aaeb9a61bdca4ab8bdc0ee318551ab52. Latest consensus state: current_term: 1 leader_uuid: "aaeb9a61bdca4ab8bdc0ee318551ab52" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aaeb9a61bdca4ab8bdc0ee318551ab52" member_type: VOTER last_known_addr { host: "127.25.108.190" port: 40189 } } peers { permanent_uuid: "0ec5572f7e8c4d27bb850d5983c8eb07" member_type: VOTER last_known_addr { host: "127.25.108.189" port: 37911 } } peers { permanent_uuid: "db0c22fe226d4071b623e4f6921f4a3d" member_type: VOTER last_known_addr { host: "127.25.108.188" port: 38177 } } }
09:04:06.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.694429 26292 sys_catalog.cc:458] T 00000000000000000000000000000000 P db0c22fe226d4071b623e4f6921f4a3d [sys.catalog]: This master's current role is: FOLLOWER
09:04:06.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.694866 26321 mvcc.cc:204] Tried to move back new op lower bound from 7216040536765861888 to 7216040536656826368. Current Snapshot: MvccSnapshot[applied={T|T < 7216040536765861888}]
09:04:06.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.696897 26293 sys_catalog.cc:455] T 00000000000000000000000000000000 P aaeb9a61bdca4ab8bdc0ee318551ab52 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "aaeb9a61bdca4ab8bdc0ee318551ab52" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aaeb9a61bdca4ab8bdc0ee318551ab52" member_type: VOTER last_known_addr { host: "127.25.108.190" port: 40189 } } peers { permanent_uuid: "0ec5572f7e8c4d27bb850d5983c8eb07" member_type: VOTER last_known_addr { host: "127.25.108.189" port: 37911 } } peers { permanent_uuid: "db0c22fe226d4071b623e4f6921f4a3d" member_type: VOTER last_known_addr { host: "127.25.108.188" port: 38177 } } }
09:04:06.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.697233 26293 sys_catalog.cc:458] T 00000000000000000000000000000000 P aaeb9a61bdca4ab8bdc0ee318551ab52 [sys.catalog]: This master's current role is: LEADER
09:04:06.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.702857 26320 catalog_manager.cc:1357] Generated new cluster ID: 8e6b4924608e40b9b93ef0e275611781
09:04:06.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.702984 26320 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
09:04:06.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.708710 26292 sys_catalog.cc:455] T 00000000000000000000000000000000 P db0c22fe226d4071b623e4f6921f4a3d [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "aaeb9a61bdca4ab8bdc0ee318551ab52" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aaeb9a61bdca4ab8bdc0ee318551ab52" member_type: VOTER last_known_addr { host: "127.25.108.190" port: 40189 } } peers { permanent_uuid: "0ec5572f7e8c4d27bb850d5983c8eb07" member_type: VOTER last_known_addr { host: "127.25.108.189" port: 37911 } } peers { permanent_uuid: "db0c22fe226d4071b623e4f6921f4a3d" member_type: VOTER last_known_addr { host: "127.25.108.188" port: 38177 } } }
09:04:06.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.709007 26292 sys_catalog.cc:458] T 00000000000000000000000000000000 P db0c22fe226d4071b623e4f6921f4a3d [sys.catalog]: This master's current role is: FOLLOWER
09:04:06.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.726467 26320 catalog_manager.cc:1380] Generated new certificate authority record
09:04:06.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.729676 26320 catalog_manager.cc:1514] Loading token signing keys...
09:04:06.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.756112 26320 catalog_manager.cc:6022] T 00000000000000000000000000000000 P aaeb9a61bdca4ab8bdc0ee318551ab52: Generated new TSK 0
09:04:06.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.757104 26202 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.108.188" port: 38177 } attempt: 5
09:04:06.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.757295 26320 catalog_manager.cc:1524] Initializing in-progress tserver states...
09:04:06.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.770251 26202 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07: Bootstrap starting.
09:04:06.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.774684 26202 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07: Neither blocks nor log segments found. Creating new log.
09:04:06.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.776227 26202 log.cc:826] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07: Log is configured to *not* fsync() on all Append() calls
09:04:06.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.778846 26034 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskescuMK/build/asan/bin/kudu
09:04:06.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskescuMK/build/asan/bin/kudu
09:04:06.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/ts-0/wal
09:04:06.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/ts-0/data
09:04:06.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:04:06.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:04:06.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:04:06.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:04:06.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:04:06.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:04:06.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/ts-0/logs
09:04:06.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/ts-0/data/info.pb
09:04:06.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:04:06.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:04:06.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:04:06.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:04:06.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:04:06.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:04:06.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:04:06.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:04:06.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
09:04:06.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:04:06.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.108.129:0
09:04:06.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.25.108.129
09:04:06.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.108.129
09:04:06.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:04:06.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.25.108.190:40189,127.25.108.189:37911,127.25.108.188:38177
09:04:06.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.108.148:42419
09:04:06.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:04:06.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:04:06.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:04:06.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
09:04:06.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
09:04:06.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
09:04:06.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.785398 26202 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07: No bootstrap required, opened a new log
09:04:06.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.793056 26202 raft_consensus.cc:359] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aaeb9a61bdca4ab8bdc0ee318551ab52" member_type: VOTER last_known_addr { host: "127.25.108.190" port: 40189 } } peers { permanent_uuid: "0ec5572f7e8c4d27bb850d5983c8eb07" member_type: VOTER last_known_addr { host: "127.25.108.189" port: 37911 } } peers { permanent_uuid: "db0c22fe226d4071b623e4f6921f4a3d" member_type: VOTER last_known_addr { host: "127.25.108.188" port: 38177 } }
09:04:06.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.793514 26202 raft_consensus.cc:385] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:04:06.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.793722 26202 raft_consensus.cc:740] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0ec5572f7e8c4d27bb850d5983c8eb07, State: Initialized, Role: FOLLOWER
09:04:06.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.794984 26202 consensus_queue.cc:260] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aaeb9a61bdca4ab8bdc0ee318551ab52" member_type: VOTER last_known_addr { host: "127.25.108.190" port: 40189 } } peers { permanent_uuid: "0ec5572f7e8c4d27bb850d5983c8eb07" member_type: VOTER last_known_addr { host: "127.25.108.189" port: 37911 } } peers { permanent_uuid: "db0c22fe226d4071b623e4f6921f4a3d" member_type: VOTER last_known_addr { host: "127.25.108.188" port: 38177 } }
09:04:06.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.802047 26328 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aaeb9a61bdca4ab8bdc0ee318551ab52" member_type: VOTER last_known_addr { host: "127.25.108.190" port: 40189 } } peers { permanent_uuid: "0ec5572f7e8c4d27bb850d5983c8eb07" member_type: VOTER last_known_addr { host: "127.25.108.189" port: 37911 } } peers { permanent_uuid: "db0c22fe226d4071b623e4f6921f4a3d" member_type: VOTER last_known_addr { host: "127.25.108.188" port: 38177 } } }
09:04:06.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.804206 26328 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07 [sys.catalog]: This master's current role is: FOLLOWER
09:04:06.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.804147 26166 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "aaeb9a61bdca4ab8bdc0ee318551ab52" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0ec5572f7e8c4d27bb850d5983c8eb07"
09:04:06.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.804761 26165 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07 [term 0 FOLLOWER]: Advancing to term 1
09:04:06.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.804073 26202 sys_catalog.cc:565] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07 [sys.catalog]: configured and running, proceeding with master startup.
09:04:06.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.808316 26165 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07 [term 1 FOLLOWER]: Refusing update from remote peer aaeb9a61bdca4ab8bdc0ee318551ab52: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
09:04:06.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.808558 26167 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "aaeb9a61bdca4ab8bdc0ee318551ab52" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0ec5572f7e8c4d27bb850d5983c8eb07" is_pre_election: true
09:04:06.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.810936 26293 consensus_queue.cc:1048] T 00000000000000000000000000000000 P aaeb9a61bdca4ab8bdc0ee318551ab52 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0ec5572f7e8c4d27bb850d5983c8eb07" member_type: VOTER last_known_addr { host: "127.25.108.189" port: 37911 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
09:04:06.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.872210 26328 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07 [sys.catalog]: SysCatalogTable state changed. Reason: New leader aaeb9a61bdca4ab8bdc0ee318551ab52. Latest consensus state: current_term: 1 leader_uuid: "aaeb9a61bdca4ab8bdc0ee318551ab52" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aaeb9a61bdca4ab8bdc0ee318551ab52" member_type: VOTER last_known_addr { host: "127.25.108.190" port: 40189 } } peers { permanent_uuid: "0ec5572f7e8c4d27bb850d5983c8eb07" member_type: VOTER last_known_addr { host: "127.25.108.189" port: 37911 } } peers { permanent_uuid: "db0c22fe226d4071b623e4f6921f4a3d" member_type: VOTER last_known_addr { host: "127.25.108.188" port: 38177 } } }
09:04:06.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.872473 26328 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07 [sys.catalog]: This master's current role is: FOLLOWER
09:04:06.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.872997 26337 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "aaeb9a61bdca4ab8bdc0ee318551ab52" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aaeb9a61bdca4ab8bdc0ee318551ab52" member_type: VOTER last_known_addr { host: "127.25.108.190" port: 40189 } } peers { permanent_uuid: "0ec5572f7e8c4d27bb850d5983c8eb07" member_type: VOTER last_known_addr { host: "127.25.108.189" port: 37911 } } peers { permanent_uuid: "db0c22fe226d4071b623e4f6921f4a3d" member_type: VOTER last_known_addr { host: "127.25.108.188" port: 38177 } } }
09:04:06.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.873397 26337 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07 [sys.catalog]: This master's current role is: FOLLOWER
09:04:06.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.883277 26345 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
09:04:06.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.928963 26293 sys_catalog.cc:455] T 00000000000000000000000000000000 P aaeb9a61bdca4ab8bdc0ee318551ab52 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "aaeb9a61bdca4ab8bdc0ee318551ab52" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aaeb9a61bdca4ab8bdc0ee318551ab52" member_type: VOTER last_known_addr { host: "127.25.108.190" port: 40189 } } peers { permanent_uuid: "0ec5572f7e8c4d27bb850d5983c8eb07" member_type: VOTER last_known_addr { host: "127.25.108.189" port: 37911 } } peers { permanent_uuid: "db0c22fe226d4071b623e4f6921f4a3d" member_type: VOTER last_known_addr { host: "127.25.108.188" port: 38177 } } }
09:04:06.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:06.929245 26293 sys_catalog.cc:458] T 00000000000000000000000000000000 P aaeb9a61bdca4ab8bdc0ee318551ab52 [sys.catalog]: This master's current role is: LEADER
09:04:07.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:07.266933 26327 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:04:07.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:07.267227 26327 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:04:07.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:07.267319 26327 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:04:07.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:07.275309 26327 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:04:07.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:07.275439 26327 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
09:04:07.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:07.275489 26327 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
09:04:07.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:07.275524 26327 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
09:04:07.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:07.275604 26327 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.25.108.129
09:04:07.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.285360 26327 tablet_server_runner.cc:78] Tablet server non-default flags:
09:04:07.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:04:07.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.108.148:42419
09:04:07.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:04:07.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:04:07.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/ts-0/data
09:04:07.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/ts-0/wal
09:04:07.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:04:07.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:04:07.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.108.129:0
09:04:07.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:04:07.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:04:07.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:04:07.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/ts-0/data/info.pb
09:04:07.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.108.129
09:04:07.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:04:07.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
09:04:07.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
09:04:07.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
09:04:07.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.25.108.190:40189,127.25.108.189:37911,127.25.108.188:38177
09:04:07.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:04:07.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:04:07.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:04:07.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:04:07.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:04:07.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.25.108.129
09:04:07.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/ts-0/logs
09:04:07.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:04:07.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:04:07.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:04:07.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
09:04:07.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:04:07.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 60f5e5267b92c39485a66121d3ce3cc7ef57b0e0
09:04:07.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:04:07.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 29 Oct 2025 08:43:21 UTC on 24a791456cd2
09:04:07.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8772
09:04:07.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
09:04:07.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.287372 26327 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:04:07.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.289306 26327 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:04:07.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:07.299932 26351 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:04:07.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:07.300182 26353 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:04:07.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:07.300684 26350 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:04:07.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.301299 26327 server_base.cc:1047] running on GCE node
09:04:07.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.301892 26327 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:04:07.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.302914 26327 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:04:07.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.304129 26327 hybrid_clock.cc:648] HybridClock initialized: now 1761728647304076 us; error 31 us; skew 500 ppm
09:04:07.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.304433 26327 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:04:07.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.306728 26327 webserver.cc:492] Webserver started at http://127.25.108.129:38453/ using document root <none> and password file <none>
09:04:07.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.307500 26327 fs_manager.cc:362] Metadata directory not provided
09:04:07.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.307633 26327 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:04:07.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.308044 26327 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
09:04:07.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.310431 26327 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/ts-0/data/instance:
09:04:07.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "06ecf9cc74fc4652a29bcab5f0d1d702"
09:04:07.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 09:04:07 on dist-test-slave-rph9"
09:04:07.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.311208 26327 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/ts-0/wal/instance:
09:04:07.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "06ecf9cc74fc4652a29bcab5f0d1d702"
09:04:07.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 09:04:07 on dist-test-slave-rph9"
09:04:07.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.316568 26327 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.007s sys 0.000s
09:04:07.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.319999 26359 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:04:07.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.321280 26327 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
09:04:07.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.321434 26327 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/ts-0/data,/tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/ts-0/wal
09:04:07.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "06ecf9cc74fc4652a29bcab5f0d1d702"
09:04:07.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 09:04:07 on dist-test-slave-rph9"
09:04:07.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.321633 26327 fs_report.cc:389] FS layout report
09:04:07.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:04:07.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/ts-0/wal
09:04:07.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/ts-0/wal
09:04:07.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/ts-0/data/data
09:04:07.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:04:07.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:04:07.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:04:07.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:04:07.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:04:07.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:04:07.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:04:07.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:04:07.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:04:07.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:04:07.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:04:07.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.350405 26327 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:04:07.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.351269 26327 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:04:07.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.351586 26327 kserver.cc:163] Server-wide thread pool size limit: 3276
09:04:07.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.354602 26327 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
09:04:07.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.354732 26327 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
09:04:07.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.354846 26327 ts_tablet_manager.cc:616] Registered 0 tablets
09:04:07.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.354970 26327 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
09:04:07.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.416292 26327 rpc_server.cc:307] RPC server started. Bound to: 127.25.108.129:37639
09:04:07.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.416348 26483 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.25.108.129:37639 every 8 connection(s)
09:04:07.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.417707 26327 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/ts-0/data/info.pb
09:04:07.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.421016 26485 heartbeater.cc:344] Connected to a master server at 127.25.108.190:40189
09:04:07.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.421392 26485 heartbeater.cc:461] Registering TS with master...
09:04:07.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.422420 26485 heartbeater.cc:507] Master 127.25.108.190:40189 requested a full tablet report, sending...
09:04:07.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.425223 26034 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskescuMK/build/asan/bin/kudu as pid 26327
09:04:07.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.425326 26075 ts_manager.cc:194] Registered new tserver with Master: 06ecf9cc74fc4652a29bcab5f0d1d702 (127.25.108.129:37639)
09:04:07.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.425421 26034 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/ts-0/wal/instance
09:04:07.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.427373 26075 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.25.108.129:49853
09:04:07.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.428071 26486 heartbeater.cc:344] Connected to a master server at 127.25.108.189:37911
09:04:07.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.428224 26486 heartbeater.cc:461] Registering TS with master...
09:04:07.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.428802 26486 heartbeater.cc:507] Master 127.25.108.189:37911 requested a full tablet report, sending...
09:04:07.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.430655 26157 ts_manager.cc:194] Registered new tserver with Master: 06ecf9cc74fc4652a29bcab5f0d1d702 (127.25.108.129:37639)
09:04:07.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.430763 26034 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskescuMK/build/asan/bin/kudu
09:04:07.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskescuMK/build/asan/bin/kudu
09:04:07.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/ts-1/wal
09:04:07.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/ts-1/data
09:04:07.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:04:07.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:04:07.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:04:07.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:04:07.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:04:07.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:04:07.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/ts-1/logs
09:04:07.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/ts-1/data/info.pb
09:04:07.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:04:07.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:04:07.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:04:07.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:04:07.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:04:07.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:04:07.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:04:07.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:04:07.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
09:04:07.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:04:07.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.108.130:0
09:04:07.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.25.108.130
09:04:07.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.108.130
09:04:07.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:04:07.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.25.108.190:40189,127.25.108.189:37911,127.25.108.188:38177
09:04:07.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.108.148:42419
09:04:07.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:04:07.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:04:07.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:04:07.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
09:04:07.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
09:04:07.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
09:04:07.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.433856 26484 heartbeater.cc:344] Connected to a master server at 127.25.108.188:38177
09:04:07.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.434128 26484 heartbeater.cc:461] Registering TS with master...
09:04:07.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.434527 26484 heartbeater.cc:507] Master 127.25.108.188:38177 requested a full tablet report, sending...
09:04:07.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.436393 26238 ts_manager.cc:194] Registered new tserver with Master: 06ecf9cc74fc4652a29bcab5f0d1d702 (127.25.108.129:37639)
09:04:07.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.588423 26306 catalog_manager.cc:1269] Loaded cluster ID: 8e6b4924608e40b9b93ef0e275611781
09:04:07.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.588516 26306 catalog_manager.cc:1562] T 00000000000000000000000000000000 P db0c22fe226d4071b623e4f6921f4a3d: loading cluster ID for follower catalog manager: success
09:04:07.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.591014 26306 catalog_manager.cc:1584] T 00000000000000000000000000000000 P db0c22fe226d4071b623e4f6921f4a3d: acquiring CA information for follower catalog manager: success
09:04:07.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.592718 26306 catalog_manager.cc:1612] T 00000000000000000000000000000000 P db0c22fe226d4071b623e4f6921f4a3d: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
09:04:07.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:07.763856 26489 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:04:07.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:07.764195 26489 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:04:07.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:07.764343 26489 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:04:07.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:07.772480 26489 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:04:07.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:07.772643 26489 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
09:04:07.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:07.772716 26489 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
09:04:07.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:07.772771 26489 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
09:04:07.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:07.772907 26489 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.25.108.130
09:04:07.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.782702 26489 tablet_server_runner.cc:78] Tablet server non-default flags:
09:04:07.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:04:07.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.108.148:42419
09:04:07.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:04:07.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:04:07.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/ts-1/data
09:04:07.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/ts-1/wal
09:04:07.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:04:07.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:04:07.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.108.130:0
09:04:07.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:04:07.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:04:07.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:04:07.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/ts-1/data/info.pb
09:04:07.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.108.130
09:04:07.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:04:07.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
09:04:07.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
09:04:07.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
09:04:07.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.25.108.190:40189,127.25.108.189:37911,127.25.108.188:38177
09:04:07.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:04:07.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:04:07.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:04:07.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:04:07.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:04:07.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.25.108.130
09:04:07.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/ts-1/logs
09:04:07.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:04:07.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:04:07.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:04:07.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
09:04:07.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:04:07.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 60f5e5267b92c39485a66121d3ce3cc7ef57b0e0
09:04:07.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:04:07.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 29 Oct 2025 08:43:21 UTC on 24a791456cd2
09:04:07.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8772
09:04:07.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
09:04:07.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.784631 26489 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:04:07.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.786391 26489 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:04:07.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:07.795858 26494 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:04:07.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:07.795913 26495 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:04:07.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:07.796772 26497 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:04:07.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.797025 26489 server_base.cc:1047] running on GCE node
09:04:07.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.797582 26489 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:04:07.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.798477 26489 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:04:07.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.799702 26489 hybrid_clock.cc:648] HybridClock initialized: now 1761728647799635 us; error 44 us; skew 500 ppm
09:04:07.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.800022 26489 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:04:07.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.802188 26489 webserver.cc:492] Webserver started at http://127.25.108.130:33977/ using document root <none> and password file <none>
09:04:07.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.802867 26489 fs_manager.cc:362] Metadata directory not provided
09:04:07.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.802965 26489 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:04:07.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.803249 26489 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
09:04:07.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.805557 26489 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/ts-1/data/instance:
09:04:07.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a2ba097a80234997bff89ca113391c46"
09:04:07.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 09:04:07 on dist-test-slave-rph9"
09:04:07.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.806253 26489 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/ts-1/wal/instance:
09:04:07.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a2ba097a80234997bff89ca113391c46"
09:04:07.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 09:04:07 on dist-test-slave-rph9"
09:04:07.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.811403 26489 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.006s sys 0.000s
09:04:07.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.814680 26503 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:04:07.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.816076 26489 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.001s sys 0.002s
09:04:07.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.816219 26489 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/ts-1/data,/tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/ts-1/wal
09:04:07.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a2ba097a80234997bff89ca113391c46"
09:04:07.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 09:04:07 on dist-test-slave-rph9"
09:04:07.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.816394 26489 fs_report.cc:389] FS layout report
09:04:07.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:04:07.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/ts-1/wal
09:04:07.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/ts-1/wal
09:04:07.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/ts-1/data/data
09:04:07.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:04:07.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:04:07.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:04:07.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:04:07.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:04:07.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:04:07.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:04:07.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:04:07.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:04:07.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:04:07.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:04:07.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.844013 26489 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:04:07.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.844852 26489 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:04:07.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.845126 26489 kserver.cc:163] Server-wide thread pool size limit: 3276
09:04:07.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.848271 26489 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
09:04:07.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.848376 26489 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
09:04:07.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.848508 26489 ts_tablet_manager.cc:616] Registered 0 tablets
09:04:07.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.848598 26489 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
09:04:07.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.895793 26345 catalog_manager.cc:1269] Loaded cluster ID: 8e6b4924608e40b9b93ef0e275611781
09:04:07.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.895954 26345 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07: loading cluster ID for follower catalog manager: success
09:04:07.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.898869 26345 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07: acquiring CA information for follower catalog manager: success
09:04:07.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.902297 26345 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
09:04:07.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.922729 26489 rpc_server.cc:307] RPC server started. Bound to: 127.25.108.130:43701
09:04:07.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.922755 26626 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.25.108.130:43701 every 8 connection(s)
09:04:07.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.924597 26489 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/ts-1/data/info.pb
09:04:07.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.929420 26628 heartbeater.cc:344] Connected to a master server at 127.25.108.190:40189
09:04:07.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.929847 26628 heartbeater.cc:461] Registering TS with master...
09:04:07.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.930549 26034 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskescuMK/build/asan/bin/kudu as pid 26489
09:04:07.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.930742 26034 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/ts-1/wal/instance
09:04:07.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.930984 26628 heartbeater.cc:507] Master 127.25.108.190:40189 requested a full tablet report, sending...
09:04:07.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.932621 26076 ts_manager.cc:194] Registered new tserver with Master: a2ba097a80234997bff89ca113391c46 (127.25.108.130:43701)
09:04:07.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.933912 26076 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.25.108.130:44851
09:04:07.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.936511 26034 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskescuMK/build/asan/bin/kudu
09:04:07.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskescuMK/build/asan/bin/kudu
09:04:07.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/ts-2/wal
09:04:07.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/ts-2/data
09:04:07.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:04:07.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:04:07.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:04:07.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:04:07.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:04:07.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:04:07.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/ts-2/logs
09:04:07.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/ts-2/data/info.pb
09:04:07.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:04:07.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:04:07.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:04:07.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:04:07.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:04:07.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:04:07.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:04:07.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:04:07.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
09:04:07.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:04:07.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.108.131:0
09:04:07.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.25.108.131
09:04:07.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.108.131
09:04:07.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:04:07.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.25.108.190:40189,127.25.108.189:37911,127.25.108.188:38177
09:04:07.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.108.148:42419
09:04:07.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:04:07.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:04:07.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:04:07.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
09:04:07.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
09:04:07.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
09:04:07.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.954392 26627 heartbeater.cc:344] Connected to a master server at 127.25.108.188:38177
09:04:07.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.954550 26629 heartbeater.cc:344] Connected to a master server at 127.25.108.189:37911
09:04:07.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.954674 26629 heartbeater.cc:461] Registering TS with master...
09:04:07.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.954679 26627 heartbeater.cc:461] Registering TS with master...
09:04:07.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.955204 26627 heartbeater.cc:507] Master 127.25.108.188:38177 requested a full tablet report, sending...
09:04:07.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.955698 26629 heartbeater.cc:507] Master 127.25.108.189:37911 requested a full tablet report, sending...
09:04:07.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.956369 26238 ts_manager.cc:194] Registered new tserver with Master: a2ba097a80234997bff89ca113391c46 (127.25.108.130:43701)
09:04:07.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:07.957656 26157 ts_manager.cc:194] Registered new tserver with Master: a2ba097a80234997bff89ca113391c46 (127.25.108.130:43701)
09:04:08.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:08.297945 26632 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:04:08.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:08.298278 26632 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:04:08.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:08.298429 26632 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:04:08.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:08.307322 26632 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:04:08.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:08.307493 26632 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
09:04:08.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:08.307564 26632 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
09:04:08.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:08.307617 26632 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
09:04:08.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:08.307751 26632 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.25.108.131
09:04:08.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.317883 26632 tablet_server_runner.cc:78] Tablet server non-default flags:
09:04:08.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:04:08.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.108.148:42419
09:04:08.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:04:08.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:04:08.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/ts-2/data
09:04:08.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/ts-2/wal
09:04:08.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:04:08.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:04:08.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.108.131:0
09:04:08.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:04:08.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:04:08.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:04:08.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/ts-2/data/info.pb
09:04:08.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.108.131
09:04:08.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:04:08.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
09:04:08.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
09:04:08.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
09:04:08.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.25.108.190:40189,127.25.108.189:37911,127.25.108.188:38177
09:04:08.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:04:08.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:04:08.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:04:08.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:04:08.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:04:08.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.25.108.131
09:04:08.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/ts-2/logs
09:04:08.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:04:08.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:04:08.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:04:08.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
09:04:08.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:04:08.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 60f5e5267b92c39485a66121d3ce3cc7ef57b0e0
09:04:08.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:04:08.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 29 Oct 2025 08:43:21 UTC on 24a791456cd2
09:04:08.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8772
09:04:08.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
09:04:08.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.320024 26632 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:04:08.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.321933 26632 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:04:08.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:08.331784 26637 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:04:08.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:08.331751 26638 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:04:08.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:08.332165 26640 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:04:08.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.333567 26632 server_base.cc:1047] running on GCE node
09:04:08.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.334174 26632 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:04:08.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.335223 26632 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:04:08.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.336436 26632 hybrid_clock.cc:648] HybridClock initialized: now 1761728648336363 us; error 45 us; skew 500 ppm
09:04:08.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.336748 26632 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:04:08.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.338948 26632 webserver.cc:492] Webserver started at http://127.25.108.131:33253/ using document root <none> and password file <none>
09:04:08.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.339676 26632 fs_manager.cc:362] Metadata directory not provided
09:04:08.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.339812 26632 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:04:08.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.340194 26632 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
09:04:08.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.342576 26632 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/ts-2/data/instance:
09:04:08.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "281228de85874ca1964dc443d6ef78ea"
09:04:08.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 09:04:08 on dist-test-slave-rph9"
09:04:08.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.343394 26632 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/ts-2/wal/instance:
09:04:08.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "281228de85874ca1964dc443d6ef78ea"
09:04:08.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 09:04:08 on dist-test-slave-rph9"
09:04:08.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.349176 26632 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.007s sys 0.000s
09:04:08.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.352662 26646 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:04:08.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.353971 26632 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
09:04:08.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.354146 26632 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/ts-2/data,/tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/ts-2/wal
09:04:08.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "281228de85874ca1964dc443d6ef78ea"
09:04:08.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 09:04:08 on dist-test-slave-rph9"
09:04:08.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.354350 26632 fs_report.cc:389] FS layout report
09:04:08.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:04:08.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/ts-2/wal
09:04:08.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/ts-2/wal
09:04:08.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/ts-2/data/data
09:04:08.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:04:08.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:04:08.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:04:08.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:04:08.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:04:08.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:04:08.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:04:08.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:04:08.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:04:08.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:04:08.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:04:08.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.377365 26632 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:04:08.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.378280 26632 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:04:08.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.378618 26632 kserver.cc:163] Server-wide thread pool size limit: 3276
09:04:08.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.382102 26632 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
09:04:08.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.382251 26632 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
09:04:08.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.382375 26632 ts_tablet_manager.cc:616] Registered 0 tablets
09:04:08.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.382453 26632 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
09:04:08.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.433193 26485 heartbeater.cc:499] Master 127.25.108.190:40189 was elected leader, sending a full tablet report...
09:04:08.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.441187 26632 rpc_server.cc:307] RPC server started. Bound to: 127.25.108.131:32837
09:04:08.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.441253 26767 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.25.108.131:32837 every 8 connection(s)
09:04:08.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.442653 26632 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/ts-2/data/info.pb
09:04:08.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.445626 26769 heartbeater.cc:344] Connected to a master server at 127.25.108.190:40189
09:04:08.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.446225 26769 heartbeater.cc:461] Registering TS with master...
09:04:08.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.447749 26769 heartbeater.cc:507] Master 127.25.108.190:40189 requested a full tablet report, sending...
09:04:08.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.449985 26076 ts_manager.cc:194] Registered new tserver with Master: 281228de85874ca1964dc443d6ef78ea (127.25.108.131:32837)
09:04:08.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.451100 26076 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.25.108.131:43981
09:04:08.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.451777 26034 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskescuMK/build/asan/bin/kudu as pid 26632
09:04:08.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.451963 26034 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskescuMK/test-tmp/mini-kudu-cluster2647032151149951194/ts-2/wal/instance
09:04:08.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.456676 26768 heartbeater.cc:344] Connected to a master server at 127.25.108.188:38177
09:04:08.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.456844 26768 heartbeater.cc:461] Registering TS with master...
09:04:08.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.458019 26768 heartbeater.cc:507] Master 127.25.108.188:38177 requested a full tablet report, sending...
09:04:08.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.459172 26238 ts_manager.cc:194] Registered new tserver with Master: 281228de85874ca1964dc443d6ef78ea (127.25.108.131:32837)
09:04:08.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.462903 26770 heartbeater.cc:344] Connected to a master server at 127.25.108.189:37911
09:04:08.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.463060 26770 heartbeater.cc:461] Registering TS with master...
09:04:08.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.463313 26770 heartbeater.cc:507] Master 127.25.108.189:37911 requested a full tablet report, sending...
09:04:08.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.464259 26157 ts_manager.cc:194] Registered new tserver with Master: 281228de85874ca1964dc443d6ef78ea (127.25.108.131:32837)
09:04:08.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.476191 26034 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
09:04:08.478 [DEBUG - main] (MiniKuduCluster.java:195) Response:
09:04:08.478 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
09:04:08.479 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.25.108.190"
port: 40189
}
bound_http_address {
host: "127.25.108.190"
port: 39581
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.25.108.189"
port: 37911
}
bound_http_address {
host: "127.25.108.189"
port: 45135
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.25.108.188"
port: 38177
}
bound_http_address {
host: "127.25.108.188"
port: 40567
}
}
}
09:04:08.479 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
09:04:08.480 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.25.108.129"
port: 37639
}
bound_http_address {
host: "127.25.108.129"
port: 38453
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.25.108.130"
port: 43701
}
bound_http_address {
host: "127.25.108.130"
port: 33977
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.25.108.131"
port: 32837
}
bound_http_address {
host: "127.25.108.131"
port: 33253
}
}
}
09:04:08.480 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
09:04:08.482 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
09:04:08.484 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.25.108.190:40189(127.25.108.190:40189)] connecting to peer
09:04:08.485 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.25.108.188:38177(127.25.108.188:38177)] connecting to peer
09:04:08.486 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.25.108.189:37911(127.25.108.189:37911)] connecting to peer
09:04:08.488 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.25.108.190:40189(127.25.108.190:40189)] Successfully connected to peer
09:04:08.489 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.25.108.189:37911(127.25.108.189:37911)] Successfully connected to peer
09:04:08.489 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.25.108.188:38177(127.25.108.188:38177)] Successfully connected to peer
09:04:08.492 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.25.108.190
09:04:08.497 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.25.108.188
09:04:08.509 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.25.108.189
09:04:08.512 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xffe6bcf3, L:/127.0.0.1:41170 - R:/127.25.108.188:38177] using SASL/PLAIN
09:04:08.515 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x690194c3, L:/127.0.0.1:60512 - R:/127.25.108.190:40189] using SASL/PLAIN
09:04:08.524 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
09:04:08.524 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=40), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.190:40189, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.188:38177, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.189:37911, [29ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.188:38177: OK, [38ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.190:40189: OK, deferred=null) after lookup
09:04:08.526 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x2d8fb33b, L:/127.0.0.1:39020 - R:/127.25.108.189:37911] using SASL/PLAIN
09:04:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.528702 26076 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:60512:
09:04:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_manager_fallback"
09:04:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
09:04:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:04:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
09:04:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
09:04:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:04:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:04:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:04:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:04:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
09:04:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:04:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column1_i"
09:04:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
09:04:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
09:04:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:04:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:04:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:04:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
09:04:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:04:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column2_i"
09:04:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
09:04:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
09:04:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:04:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:04:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:04:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
09:04:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:04:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column3_s"
09:04:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
09:04:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
09:04:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
09:04:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: DICT_ENCODING
09:04:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: LZ4
09:04:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 4096
09:04:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:04:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
09:04:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:04:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column4_b"
09:04:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: BOOL
09:04:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
09:04:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:04:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:04:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:04:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
09:04:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
09:04:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) hash_schema {
09:04:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:04:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
09:04:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_buckets: 2
09:04:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) seed: 0
09:04:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:08.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:08.531098 26076 catalog_manager.cc:7011] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table txn_manager_fallback in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
09:04:08.548 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 1461c31339b24f7783bf16fa929cf842
09:04:08.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.593533 26418 tablet_service.cc:1505] Processing CreateTablet for tablet 8bf23027b1c64e248061f4f4a34dc7a6 (DEFAULT_TABLE table=txn_manager_fallback [id=1461c31339b24f7783bf16fa929cf842]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
09:04:08.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.593479 26562 tablet_service.cc:1505] Processing CreateTablet for tablet 474f7edac15d4c468a54a651ead3a2a5 (DEFAULT_TABLE table=txn_manager_fallback [id=1461c31339b24f7783bf16fa929cf842]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
09:04:08.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.593793 26702 tablet_service.cc:1505] Processing CreateTablet for tablet 8bf23027b1c64e248061f4f4a34dc7a6 (DEFAULT_TABLE table=txn_manager_fallback [id=1461c31339b24f7783bf16fa929cf842]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
09:04:08.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.594476 26561 tablet_service.cc:1505] Processing CreateTablet for tablet 8bf23027b1c64e248061f4f4a34dc7a6 (DEFAULT_TABLE table=txn_manager_fallback [id=1461c31339b24f7783bf16fa929cf842]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
09:04:08.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.594941 26418 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8bf23027b1c64e248061f4f4a34dc7a6. 1 dirs total, 0 dirs full, 0 dirs failed
09:04:08.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.594950 26702 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8bf23027b1c64e248061f4f4a34dc7a6. 1 dirs total, 0 dirs full, 0 dirs failed
09:04:08.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.594952 26562 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 474f7edac15d4c468a54a651ead3a2a5. 1 dirs total, 0 dirs full, 0 dirs failed
09:04:08.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.593814 26703 tablet_service.cc:1505] Processing CreateTablet for tablet 474f7edac15d4c468a54a651ead3a2a5 (DEFAULT_TABLE table=txn_manager_fallback [id=1461c31339b24f7783bf16fa929cf842]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
09:04:08.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.595665 26703 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 474f7edac15d4c468a54a651ead3a2a5. 1 dirs total, 0 dirs full, 0 dirs failed
09:04:08.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.597396 26561 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8bf23027b1c64e248061f4f4a34dc7a6. 1 dirs total, 0 dirs full, 0 dirs failed
09:04:08.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.593485 26419 tablet_service.cc:1505] Processing CreateTablet for tablet 474f7edac15d4c468a54a651ead3a2a5 (DEFAULT_TABLE table=txn_manager_fallback [id=1461c31339b24f7783bf16fa929cf842]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
09:04:08.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.598397 26419 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 474f7edac15d4c468a54a651ead3a2a5. 1 dirs total, 0 dirs full, 0 dirs failed
09:04:08.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.615576 26794 tablet_bootstrap.cc:492] T 474f7edac15d4c468a54a651ead3a2a5 P 06ecf9cc74fc4652a29bcab5f0d1d702: Bootstrap starting.
09:04:08.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.615621 26796 tablet_bootstrap.cc:492] T 474f7edac15d4c468a54a651ead3a2a5 P a2ba097a80234997bff89ca113391c46: Bootstrap starting.
09:04:08.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.619185 26796 tablet_bootstrap.cc:654] T 474f7edac15d4c468a54a651ead3a2a5 P a2ba097a80234997bff89ca113391c46: Neither blocks nor log segments found. Creating new log.
09:04:08.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.620153 26794 tablet_bootstrap.cc:654] T 474f7edac15d4c468a54a651ead3a2a5 P 06ecf9cc74fc4652a29bcab5f0d1d702: Neither blocks nor log segments found. Creating new log.
09:04:08.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.621548 26794 log.cc:826] T 474f7edac15d4c468a54a651ead3a2a5 P 06ecf9cc74fc4652a29bcab5f0d1d702: Log is configured to *not* fsync() on all Append() calls
09:04:08.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.621595 26796 log.cc:826] T 474f7edac15d4c468a54a651ead3a2a5 P a2ba097a80234997bff89ca113391c46: Log is configured to *not* fsync() on all Append() calls
09:04:08.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.622363 26795 tablet_bootstrap.cc:492] T 8bf23027b1c64e248061f4f4a34dc7a6 P 281228de85874ca1964dc443d6ef78ea: Bootstrap starting.
09:04:08.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.626361 26795 tablet_bootstrap.cc:654] T 8bf23027b1c64e248061f4f4a34dc7a6 P 281228de85874ca1964dc443d6ef78ea: Neither blocks nor log segments found. Creating new log.
09:04:08.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.626608 26794 tablet_bootstrap.cc:492] T 474f7edac15d4c468a54a651ead3a2a5 P 06ecf9cc74fc4652a29bcab5f0d1d702: No bootstrap required, opened a new log
09:04:08.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.626917 26794 ts_tablet_manager.cc:1403] T 474f7edac15d4c468a54a651ead3a2a5 P 06ecf9cc74fc4652a29bcab5f0d1d702: Time spent bootstrapping tablet: real 0.012s user 0.006s sys 0.003s
09:04:08.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.630626 26796 tablet_bootstrap.cc:492] T 474f7edac15d4c468a54a651ead3a2a5 P a2ba097a80234997bff89ca113391c46: No bootstrap required, opened a new log
09:04:08.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.630941 26796 ts_tablet_manager.cc:1403] T 474f7edac15d4c468a54a651ead3a2a5 P a2ba097a80234997bff89ca113391c46: Time spent bootstrapping tablet: real 0.016s user 0.008s sys 0.000s
09:04:08.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.631336 26795 log.cc:826] T 8bf23027b1c64e248061f4f4a34dc7a6 P 281228de85874ca1964dc443d6ef78ea: Log is configured to *not* fsync() on all Append() calls
09:04:08.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.633857 26794 raft_consensus.cc:359] T 474f7edac15d4c468a54a651ead3a2a5 P 06ecf9cc74fc4652a29bcab5f0d1d702 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "281228de85874ca1964dc443d6ef78ea" member_type: VOTER last_known_addr { host: "127.25.108.131" port: 32837 } } peers { permanent_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" member_type: VOTER last_known_addr { host: "127.25.108.129" port: 37639 } } peers { permanent_uuid: "a2ba097a80234997bff89ca113391c46" member_type: VOTER last_known_addr { host: "127.25.108.130" port: 43701 } }
09:04:08.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.634379 26794 raft_consensus.cc:385] T 474f7edac15d4c468a54a651ead3a2a5 P 06ecf9cc74fc4652a29bcab5f0d1d702 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:04:08.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.634505 26794 raft_consensus.cc:740] T 474f7edac15d4c468a54a651ead3a2a5 P 06ecf9cc74fc4652a29bcab5f0d1d702 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 06ecf9cc74fc4652a29bcab5f0d1d702, State: Initialized, Role: FOLLOWER
09:04:08.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.635450 26794 consensus_queue.cc:260] T 474f7edac15d4c468a54a651ead3a2a5 P 06ecf9cc74fc4652a29bcab5f0d1d702 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "281228de85874ca1964dc443d6ef78ea" member_type: VOTER last_known_addr { host: "127.25.108.131" port: 32837 } } peers { permanent_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" member_type: VOTER last_known_addr { host: "127.25.108.129" port: 37639 } } peers { permanent_uuid: "a2ba097a80234997bff89ca113391c46" member_type: VOTER last_known_addr { host: "127.25.108.130" port: 43701 } }
09:04:08.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.638587 26796 raft_consensus.cc:359] T 474f7edac15d4c468a54a651ead3a2a5 P a2ba097a80234997bff89ca113391c46 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "281228de85874ca1964dc443d6ef78ea" member_type: VOTER last_known_addr { host: "127.25.108.131" port: 32837 } } peers { permanent_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" member_type: VOTER last_known_addr { host: "127.25.108.129" port: 37639 } } peers { permanent_uuid: "a2ba097a80234997bff89ca113391c46" member_type: VOTER last_known_addr { host: "127.25.108.130" port: 43701 } }
09:04:08.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.639743 26796 raft_consensus.cc:385] T 474f7edac15d4c468a54a651ead3a2a5 P a2ba097a80234997bff89ca113391c46 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:04:08.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.639967 26796 raft_consensus.cc:740] T 474f7edac15d4c468a54a651ead3a2a5 P a2ba097a80234997bff89ca113391c46 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a2ba097a80234997bff89ca113391c46, State: Initialized, Role: FOLLOWER
09:04:08.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.641201 26796 consensus_queue.cc:260] T 474f7edac15d4c468a54a651ead3a2a5 P a2ba097a80234997bff89ca113391c46 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "281228de85874ca1964dc443d6ef78ea" member_type: VOTER last_known_addr { host: "127.25.108.131" port: 32837 } } peers { permanent_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" member_type: VOTER last_known_addr { host: "127.25.108.129" port: 37639 } } peers { permanent_uuid: "a2ba097a80234997bff89ca113391c46" member_type: VOTER last_known_addr { host: "127.25.108.130" port: 43701 } }
09:04:08.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.641484 26795 tablet_bootstrap.cc:492] T 8bf23027b1c64e248061f4f4a34dc7a6 P 281228de85874ca1964dc443d6ef78ea: No bootstrap required, opened a new log
09:04:08.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.641793 26795 ts_tablet_manager.cc:1403] T 8bf23027b1c64e248061f4f4a34dc7a6 P 281228de85874ca1964dc443d6ef78ea: Time spent bootstrapping tablet: real 0.020s user 0.006s sys 0.008s
09:04:08.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.642261 26794 ts_tablet_manager.cc:1434] T 474f7edac15d4c468a54a651ead3a2a5 P 06ecf9cc74fc4652a29bcab5f0d1d702: Time spent starting tablet: real 0.015s user 0.009s sys 0.004s
09:04:08.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.642707 26794 tablet_bootstrap.cc:492] T 8bf23027b1c64e248061f4f4a34dc7a6 P 06ecf9cc74fc4652a29bcab5f0d1d702: Bootstrap starting.
09:04:08.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.644512 26794 tablet_bootstrap.cc:654] T 8bf23027b1c64e248061f4f4a34dc7a6 P 06ecf9cc74fc4652a29bcab5f0d1d702: Neither blocks nor log segments found. Creating new log.
09:04:08.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.649876 26796 ts_tablet_manager.cc:1434] T 474f7edac15d4c468a54a651ead3a2a5 P a2ba097a80234997bff89ca113391c46: Time spent starting tablet: real 0.019s user 0.009s sys 0.006s
09:04:08.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.650465 26628 heartbeater.cc:499] Master 127.25.108.190:40189 was elected leader, sending a full tablet report...
09:04:08.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.654656 26796 tablet_bootstrap.cc:492] T 8bf23027b1c64e248061f4f4a34dc7a6 P a2ba097a80234997bff89ca113391c46: Bootstrap starting.
09:04:08.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.654984 26795 raft_consensus.cc:359] T 8bf23027b1c64e248061f4f4a34dc7a6 P 281228de85874ca1964dc443d6ef78ea [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" member_type: VOTER last_known_addr { host: "127.25.108.129" port: 37639 } } peers { permanent_uuid: "a2ba097a80234997bff89ca113391c46" member_type: VOTER last_known_addr { host: "127.25.108.130" port: 43701 } } peers { permanent_uuid: "281228de85874ca1964dc443d6ef78ea" member_type: VOTER last_known_addr { host: "127.25.108.131" port: 32837 } }
09:04:08.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.655550 26795 raft_consensus.cc:385] T 8bf23027b1c64e248061f4f4a34dc7a6 P 281228de85874ca1964dc443d6ef78ea [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:04:08.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.655692 26795 raft_consensus.cc:740] T 8bf23027b1c64e248061f4f4a34dc7a6 P 281228de85874ca1964dc443d6ef78ea [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 281228de85874ca1964dc443d6ef78ea, State: Initialized, Role: FOLLOWER
09:04:08.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.656693 26795 consensus_queue.cc:260] T 8bf23027b1c64e248061f4f4a34dc7a6 P 281228de85874ca1964dc443d6ef78ea [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" member_type: VOTER last_known_addr { host: "127.25.108.129" port: 37639 } } peers { permanent_uuid: "a2ba097a80234997bff89ca113391c46" member_type: VOTER last_known_addr { host: "127.25.108.130" port: 43701 } } peers { permanent_uuid: "281228de85874ca1964dc443d6ef78ea" member_type: VOTER last_known_addr { host: "127.25.108.131" port: 32837 } }
09:04:08.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.657421 26796 tablet_bootstrap.cc:654] T 8bf23027b1c64e248061f4f4a34dc7a6 P a2ba097a80234997bff89ca113391c46: Neither blocks nor log segments found. Creating new log.
09:04:08.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.661891 26794 tablet_bootstrap.cc:492] T 8bf23027b1c64e248061f4f4a34dc7a6 P 06ecf9cc74fc4652a29bcab5f0d1d702: No bootstrap required, opened a new log
09:04:08.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.662081 26794 ts_tablet_manager.cc:1403] T 8bf23027b1c64e248061f4f4a34dc7a6 P 06ecf9cc74fc4652a29bcab5f0d1d702: Time spent bootstrapping tablet: real 0.019s user 0.010s sys 0.007s
09:04:08.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.662451 26795 ts_tablet_manager.cc:1434] T 8bf23027b1c64e248061f4f4a34dc7a6 P 281228de85874ca1964dc443d6ef78ea: Time spent starting tablet: real 0.021s user 0.010s sys 0.005s
09:04:08.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.662631 26796 tablet_bootstrap.cc:492] T 8bf23027b1c64e248061f4f4a34dc7a6 P a2ba097a80234997bff89ca113391c46: No bootstrap required, opened a new log
09:04:08.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.662781 26769 heartbeater.cc:499] Master 127.25.108.190:40189 was elected leader, sending a full tablet report...
09:04:08.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.662875 26796 ts_tablet_manager.cc:1403] T 8bf23027b1c64e248061f4f4a34dc7a6 P a2ba097a80234997bff89ca113391c46: Time spent bootstrapping tablet: real 0.008s user 0.002s sys 0.004s
09:04:08.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.663084 26794 raft_consensus.cc:359] T 8bf23027b1c64e248061f4f4a34dc7a6 P 06ecf9cc74fc4652a29bcab5f0d1d702 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" member_type: VOTER last_known_addr { host: "127.25.108.129" port: 37639 } } peers { permanent_uuid: "a2ba097a80234997bff89ca113391c46" member_type: VOTER last_known_addr { host: "127.25.108.130" port: 43701 } } peers { permanent_uuid: "281228de85874ca1964dc443d6ef78ea" member_type: VOTER last_known_addr { host: "127.25.108.131" port: 32837 } }
09:04:08.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.663256 26794 raft_consensus.cc:385] T 8bf23027b1c64e248061f4f4a34dc7a6 P 06ecf9cc74fc4652a29bcab5f0d1d702 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:04:08.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.663338 26794 raft_consensus.cc:740] T 8bf23027b1c64e248061f4f4a34dc7a6 P 06ecf9cc74fc4652a29bcab5f0d1d702 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 06ecf9cc74fc4652a29bcab5f0d1d702, State: Initialized, Role: FOLLOWER
09:04:08.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.663586 26794 consensus_queue.cc:260] T 8bf23027b1c64e248061f4f4a34dc7a6 P 06ecf9cc74fc4652a29bcab5f0d1d702 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" member_type: VOTER last_known_addr { host: "127.25.108.129" port: 37639 } } peers { permanent_uuid: "a2ba097a80234997bff89ca113391c46" member_type: VOTER last_known_addr { host: "127.25.108.130" port: 43701 } } peers { permanent_uuid: "281228de85874ca1964dc443d6ef78ea" member_type: VOTER last_known_addr { host: "127.25.108.131" port: 32837 } }
09:04:08.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.663755 26796 raft_consensus.cc:359] T 8bf23027b1c64e248061f4f4a34dc7a6 P a2ba097a80234997bff89ca113391c46 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" member_type: VOTER last_known_addr { host: "127.25.108.129" port: 37639 } } peers { permanent_uuid: "a2ba097a80234997bff89ca113391c46" member_type: VOTER last_known_addr { host: "127.25.108.130" port: 43701 } } peers { permanent_uuid: "281228de85874ca1964dc443d6ef78ea" member_type: VOTER last_known_addr { host: "127.25.108.131" port: 32837 } }
09:04:08.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.663944 26796 raft_consensus.cc:385] T 8bf23027b1c64e248061f4f4a34dc7a6 P a2ba097a80234997bff89ca113391c46 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:04:08.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.664050 26796 raft_consensus.cc:740] T 8bf23027b1c64e248061f4f4a34dc7a6 P a2ba097a80234997bff89ca113391c46 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a2ba097a80234997bff89ca113391c46, State: Initialized, Role: FOLLOWER
09:04:08.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.664181 26794 ts_tablet_manager.cc:1434] T 8bf23027b1c64e248061f4f4a34dc7a6 P 06ecf9cc74fc4652a29bcab5f0d1d702: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
09:04:08.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.664290 26796 consensus_queue.cc:260] T 8bf23027b1c64e248061f4f4a34dc7a6 P a2ba097a80234997bff89ca113391c46 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" member_type: VOTER last_known_addr { host: "127.25.108.129" port: 37639 } } peers { permanent_uuid: "a2ba097a80234997bff89ca113391c46" member_type: VOTER last_known_addr { host: "127.25.108.130" port: 43701 } } peers { permanent_uuid: "281228de85874ca1964dc443d6ef78ea" member_type: VOTER last_known_addr { host: "127.25.108.131" port: 32837 } }
09:04:08.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.666136 26795 tablet_bootstrap.cc:492] T 474f7edac15d4c468a54a651ead3a2a5 P 281228de85874ca1964dc443d6ef78ea: Bootstrap starting.
09:04:08.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.668259 26795 tablet_bootstrap.cc:654] T 474f7edac15d4c468a54a651ead3a2a5 P 281228de85874ca1964dc443d6ef78ea: Neither blocks nor log segments found. Creating new log.
09:04:08.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.671036 26795 tablet_bootstrap.cc:492] T 474f7edac15d4c468a54a651ead3a2a5 P 281228de85874ca1964dc443d6ef78ea: No bootstrap required, opened a new log
09:04:08.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.671193 26795 ts_tablet_manager.cc:1403] T 474f7edac15d4c468a54a651ead3a2a5 P 281228de85874ca1964dc443d6ef78ea: Time spent bootstrapping tablet: real 0.005s user 0.004s sys 0.000s
09:04:08.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.671216 26796 ts_tablet_manager.cc:1434] T 8bf23027b1c64e248061f4f4a34dc7a6 P a2ba097a80234997bff89ca113391c46: Time spent starting tablet: real 0.008s user 0.002s sys 0.000s
09:04:08.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.671972 26795 raft_consensus.cc:359] T 474f7edac15d4c468a54a651ead3a2a5 P 281228de85874ca1964dc443d6ef78ea [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "281228de85874ca1964dc443d6ef78ea" member_type: VOTER last_known_addr { host: "127.25.108.131" port: 32837 } } peers { permanent_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" member_type: VOTER last_known_addr { host: "127.25.108.129" port: 37639 } } peers { permanent_uuid: "a2ba097a80234997bff89ca113391c46" member_type: VOTER last_known_addr { host: "127.25.108.130" port: 43701 } }
09:04:08.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:08.672075 26487 tablet.cc:2378] T 474f7edac15d4c468a54a651ead3a2a5 P 06ecf9cc74fc4652a29bcab5f0d1d702: Can't schedule compaction. Clean time has not been advanced past its initial value.
09:04:08.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.672163 26795 raft_consensus.cc:385] T 474f7edac15d4c468a54a651ead3a2a5 P 281228de85874ca1964dc443d6ef78ea [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:04:08.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.672840 26795 raft_consensus.cc:740] T 474f7edac15d4c468a54a651ead3a2a5 P 281228de85874ca1964dc443d6ef78ea [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 281228de85874ca1964dc443d6ef78ea, State: Initialized, Role: FOLLOWER
09:04:08.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.673084 26795 consensus_queue.cc:260] T 474f7edac15d4c468a54a651ead3a2a5 P 281228de85874ca1964dc443d6ef78ea [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "281228de85874ca1964dc443d6ef78ea" member_type: VOTER last_known_addr { host: "127.25.108.131" port: 32837 } } peers { permanent_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" member_type: VOTER last_known_addr { host: "127.25.108.129" port: 37639 } } peers { permanent_uuid: "a2ba097a80234997bff89ca113391c46" member_type: VOTER last_known_addr { host: "127.25.108.130" port: 43701 } }
09:04:08.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.673646 26795 ts_tablet_manager.cc:1434] T 474f7edac15d4c468a54a651ead3a2a5 P 281228de85874ca1964dc443d6ef78ea: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
09:04:08.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.688673 26800 raft_consensus.cc:493] T 8bf23027b1c64e248061f4f4a34dc7a6 P 06ecf9cc74fc4652a29bcab5f0d1d702 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
09:04:08.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.688925 26800 raft_consensus.cc:515] T 8bf23027b1c64e248061f4f4a34dc7a6 P 06ecf9cc74fc4652a29bcab5f0d1d702 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" member_type: VOTER last_known_addr { host: "127.25.108.129" port: 37639 } } peers { permanent_uuid: "a2ba097a80234997bff89ca113391c46" member_type: VOTER last_known_addr { host: "127.25.108.130" port: 43701 } } peers { permanent_uuid: "281228de85874ca1964dc443d6ef78ea" member_type: VOTER last_known_addr { host: "127.25.108.131" port: 32837 } }
09:04:08.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.690562 26800 leader_election.cc:290] T 8bf23027b1c64e248061f4f4a34dc7a6 P 06ecf9cc74fc4652a29bcab5f0d1d702 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a2ba097a80234997bff89ca113391c46 (127.25.108.130:43701), 281228de85874ca1964dc443d6ef78ea (127.25.108.131:32837)
09:04:08.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.697317 26582 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "8bf23027b1c64e248061f4f4a34dc7a6" candidate_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a2ba097a80234997bff89ca113391c46" is_pre_election: true
09:04:08.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.697829 26582 raft_consensus.cc:2468] T 8bf23027b1c64e248061f4f4a34dc7a6 P a2ba097a80234997bff89ca113391c46 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 06ecf9cc74fc4652a29bcab5f0d1d702 in term 0.
09:04:08.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.697751 26723 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "8bf23027b1c64e248061f4f4a34dc7a6" candidate_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "281228de85874ca1964dc443d6ef78ea" is_pre_election: true
09:04:08.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.698172 26723 raft_consensus.cc:2468] T 8bf23027b1c64e248061f4f4a34dc7a6 P 281228de85874ca1964dc443d6ef78ea [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 06ecf9cc74fc4652a29bcab5f0d1d702 in term 0.
09:04:08.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.698495 26363 leader_election.cc:304] T 8bf23027b1c64e248061f4f4a34dc7a6 P 06ecf9cc74fc4652a29bcab5f0d1d702 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 06ecf9cc74fc4652a29bcab5f0d1d702, a2ba097a80234997bff89ca113391c46; no voters:
09:04:08.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.698845 26800 raft_consensus.cc:2804] T 8bf23027b1c64e248061f4f4a34dc7a6 P 06ecf9cc74fc4652a29bcab5f0d1d702 [term 0 FOLLOWER]: Leader pre-election won for term 1
09:04:08.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.698989 26800 raft_consensus.cc:493] T 8bf23027b1c64e248061f4f4a34dc7a6 P 06ecf9cc74fc4652a29bcab5f0d1d702 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
09:04:08.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.699100 26800 raft_consensus.cc:3060] T 8bf23027b1c64e248061f4f4a34dc7a6 P 06ecf9cc74fc4652a29bcab5f0d1d702 [term 0 FOLLOWER]: Advancing to term 1
09:04:08.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:08.700911 26771 tablet.cc:2378] T 474f7edac15d4c468a54a651ead3a2a5 P 281228de85874ca1964dc443d6ef78ea: Can't schedule compaction. Clean time has not been advanced past its initial value.
09:04:08.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.701476 26800 raft_consensus.cc:515] T 8bf23027b1c64e248061f4f4a34dc7a6 P 06ecf9cc74fc4652a29bcab5f0d1d702 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" member_type: VOTER last_known_addr { host: "127.25.108.129" port: 37639 } } peers { permanent_uuid: "a2ba097a80234997bff89ca113391c46" member_type: VOTER last_known_addr { host: "127.25.108.130" port: 43701 } } peers { permanent_uuid: "281228de85874ca1964dc443d6ef78ea" member_type: VOTER last_known_addr { host: "127.25.108.131" port: 32837 } }
09:04:08.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.702095 26800 leader_election.cc:290] T 8bf23027b1c64e248061f4f4a34dc7a6 P 06ecf9cc74fc4652a29bcab5f0d1d702 [CANDIDATE]: Term 1 election: Requested vote from peers a2ba097a80234997bff89ca113391c46 (127.25.108.130:43701), 281228de85874ca1964dc443d6ef78ea (127.25.108.131:32837)
09:04:08.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.702538 26582 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "8bf23027b1c64e248061f4f4a34dc7a6" candidate_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a2ba097a80234997bff89ca113391c46"
09:04:08.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.702600 26723 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "8bf23027b1c64e248061f4f4a34dc7a6" candidate_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "281228de85874ca1964dc443d6ef78ea"
09:04:08.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.702737 26723 raft_consensus.cc:3060] T 8bf23027b1c64e248061f4f4a34dc7a6 P 281228de85874ca1964dc443d6ef78ea [term 0 FOLLOWER]: Advancing to term 1
09:04:08.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.702711 26582 raft_consensus.cc:3060] T 8bf23027b1c64e248061f4f4a34dc7a6 P a2ba097a80234997bff89ca113391c46 [term 0 FOLLOWER]: Advancing to term 1
09:04:08.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.704866 26723 raft_consensus.cc:2468] T 8bf23027b1c64e248061f4f4a34dc7a6 P 281228de85874ca1964dc443d6ef78ea [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 06ecf9cc74fc4652a29bcab5f0d1d702 in term 1.
09:04:08.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.704906 26582 raft_consensus.cc:2468] T 8bf23027b1c64e248061f4f4a34dc7a6 P a2ba097a80234997bff89ca113391c46 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 06ecf9cc74fc4652a29bcab5f0d1d702 in term 1.
09:04:08.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.705236 26363 leader_election.cc:304] T 8bf23027b1c64e248061f4f4a34dc7a6 P 06ecf9cc74fc4652a29bcab5f0d1d702 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 06ecf9cc74fc4652a29bcab5f0d1d702, 281228de85874ca1964dc443d6ef78ea; no voters:
09:04:08.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.705511 26800 raft_consensus.cc:2804] T 8bf23027b1c64e248061f4f4a34dc7a6 P 06ecf9cc74fc4652a29bcab5f0d1d702 [term 1 FOLLOWER]: Leader election won for term 1
09:04:08.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.705905 26800 raft_consensus.cc:697] T 8bf23027b1c64e248061f4f4a34dc7a6 P 06ecf9cc74fc4652a29bcab5f0d1d702 [term 1 LEADER]: Becoming Leader. State: Replica: 06ecf9cc74fc4652a29bcab5f0d1d702, State: Running, Role: LEADER
09:04:08.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.706411 26800 consensus_queue.cc:237] T 8bf23027b1c64e248061f4f4a34dc7a6 P 06ecf9cc74fc4652a29bcab5f0d1d702 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" member_type: VOTER last_known_addr { host: "127.25.108.129" port: 37639 } } peers { permanent_uuid: "a2ba097a80234997bff89ca113391c46" member_type: VOTER last_known_addr { host: "127.25.108.130" port: 43701 } } peers { permanent_uuid: "281228de85874ca1964dc443d6ef78ea" member_type: VOTER last_known_addr { host: "127.25.108.131" port: 32837 } }
09:04:08.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.710850 26076 catalog_manager.cc:5649] T 8bf23027b1c64e248061f4f4a34dc7a6 P 06ecf9cc74fc4652a29bcab5f0d1d702 reported cstate change: term changed from 0 to 1, leader changed from <none> to 06ecf9cc74fc4652a29bcab5f0d1d702 (127.25.108.129). New cstate: current_term: 1 leader_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" member_type: VOTER last_known_addr { host: "127.25.108.129" port: 37639 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a2ba097a80234997bff89ca113391c46" member_type: VOTER last_known_addr { host: "127.25.108.130" port: 43701 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "281228de85874ca1964dc443d6ef78ea" member_type: VOTER last_known_addr { host: "127.25.108.131" port: 32837 } health_report { overall_health: UNKNOWN } } }
09:04:08.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:08.719266 26630 tablet.cc:2378] T 474f7edac15d4c468a54a651ead3a2a5 P a2ba097a80234997bff89ca113391c46: Can't schedule compaction. Clean time has not been advanced past its initial value.
09:04:08.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.823979 26801 raft_consensus.cc:493] T 474f7edac15d4c468a54a651ead3a2a5 P a2ba097a80234997bff89ca113391c46 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
09:04:08.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.824210 26801 raft_consensus.cc:515] T 474f7edac15d4c468a54a651ead3a2a5 P a2ba097a80234997bff89ca113391c46 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "281228de85874ca1964dc443d6ef78ea" member_type: VOTER last_known_addr { host: "127.25.108.131" port: 32837 } } peers { permanent_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" member_type: VOTER last_known_addr { host: "127.25.108.129" port: 37639 } } peers { permanent_uuid: "a2ba097a80234997bff89ca113391c46" member_type: VOTER last_known_addr { host: "127.25.108.130" port: 43701 } }
09:04:08.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.826007 26801 leader_election.cc:290] T 474f7edac15d4c468a54a651ead3a2a5 P a2ba097a80234997bff89ca113391c46 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 281228de85874ca1964dc443d6ef78ea (127.25.108.131:32837), 06ecf9cc74fc4652a29bcab5f0d1d702 (127.25.108.129:37639)
09:04:08.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.832967 26723 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "474f7edac15d4c468a54a651ead3a2a5" candidate_uuid: "a2ba097a80234997bff89ca113391c46" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "281228de85874ca1964dc443d6ef78ea" is_pre_election: true
09:04:08.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.833004 26439 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "474f7edac15d4c468a54a651ead3a2a5" candidate_uuid: "a2ba097a80234997bff89ca113391c46" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" is_pre_election: true
09:04:08.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.833285 26723 raft_consensus.cc:2468] T 474f7edac15d4c468a54a651ead3a2a5 P 281228de85874ca1964dc443d6ef78ea [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a2ba097a80234997bff89ca113391c46 in term 0.
09:04:08.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.833396 26439 raft_consensus.cc:2468] T 474f7edac15d4c468a54a651ead3a2a5 P 06ecf9cc74fc4652a29bcab5f0d1d702 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a2ba097a80234997bff89ca113391c46 in term 0.
09:04:08.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.833886 26507 leader_election.cc:304] T 474f7edac15d4c468a54a651ead3a2a5 P a2ba097a80234997bff89ca113391c46 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 3 yes votes; 0 no votes. yes voters: 06ecf9cc74fc4652a29bcab5f0d1d702, 281228de85874ca1964dc443d6ef78ea, a2ba097a80234997bff89ca113391c46; no voters:
09:04:08.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.834309 26801 raft_consensus.cc:2804] T 474f7edac15d4c468a54a651ead3a2a5 P a2ba097a80234997bff89ca113391c46 [term 0 FOLLOWER]: Leader pre-election won for term 1
09:04:08.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.834424 26801 raft_consensus.cc:493] T 474f7edac15d4c468a54a651ead3a2a5 P a2ba097a80234997bff89ca113391c46 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
09:04:08.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.834491 26801 raft_consensus.cc:3060] T 474f7edac15d4c468a54a651ead3a2a5 P a2ba097a80234997bff89ca113391c46 [term 0 FOLLOWER]: Advancing to term 1
09:04:08.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.836858 26801 raft_consensus.cc:515] T 474f7edac15d4c468a54a651ead3a2a5 P a2ba097a80234997bff89ca113391c46 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "281228de85874ca1964dc443d6ef78ea" member_type: VOTER last_known_addr { host: "127.25.108.131" port: 32837 } } peers { permanent_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" member_type: VOTER last_known_addr { host: "127.25.108.129" port: 37639 } } peers { permanent_uuid: "a2ba097a80234997bff89ca113391c46" member_type: VOTER last_known_addr { host: "127.25.108.130" port: 43701 } }
09:04:08.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.837404 26801 leader_election.cc:290] T 474f7edac15d4c468a54a651ead3a2a5 P a2ba097a80234997bff89ca113391c46 [CANDIDATE]: Term 1 election: Requested vote from peers 281228de85874ca1964dc443d6ef78ea (127.25.108.131:32837), 06ecf9cc74fc4652a29bcab5f0d1d702 (127.25.108.129:37639)
09:04:08.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.837893 26439 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "474f7edac15d4c468a54a651ead3a2a5" candidate_uuid: "a2ba097a80234997bff89ca113391c46" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702"
09:04:08.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.837893 26723 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "474f7edac15d4c468a54a651ead3a2a5" candidate_uuid: "a2ba097a80234997bff89ca113391c46" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "281228de85874ca1964dc443d6ef78ea"
09:04:08.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.838099 26439 raft_consensus.cc:3060] T 474f7edac15d4c468a54a651ead3a2a5 P 06ecf9cc74fc4652a29bcab5f0d1d702 [term 0 FOLLOWER]: Advancing to term 1
09:04:08.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.838119 26723 raft_consensus.cc:3060] T 474f7edac15d4c468a54a651ead3a2a5 P 281228de85874ca1964dc443d6ef78ea [term 0 FOLLOWER]: Advancing to term 1
09:04:08.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.841194 26723 raft_consensus.cc:2468] T 474f7edac15d4c468a54a651ead3a2a5 P 281228de85874ca1964dc443d6ef78ea [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a2ba097a80234997bff89ca113391c46 in term 1.
09:04:08.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.841238 26439 raft_consensus.cc:2468] T 474f7edac15d4c468a54a651ead3a2a5 P 06ecf9cc74fc4652a29bcab5f0d1d702 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a2ba097a80234997bff89ca113391c46 in term 1.
09:04:08.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.841635 26505 leader_election.cc:304] T 474f7edac15d4c468a54a651ead3a2a5 P a2ba097a80234997bff89ca113391c46 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 06ecf9cc74fc4652a29bcab5f0d1d702, a2ba097a80234997bff89ca113391c46; no voters:
09:04:08.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.841924 26801 raft_consensus.cc:2804] T 474f7edac15d4c468a54a651ead3a2a5 P a2ba097a80234997bff89ca113391c46 [term 1 FOLLOWER]: Leader election won for term 1
09:04:08.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.842363 26801 raft_consensus.cc:697] T 474f7edac15d4c468a54a651ead3a2a5 P a2ba097a80234997bff89ca113391c46 [term 1 LEADER]: Becoming Leader. State: Replica: a2ba097a80234997bff89ca113391c46, State: Running, Role: LEADER
09:04:08.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.842850 26801 consensus_queue.cc:237] T 474f7edac15d4c468a54a651ead3a2a5 P a2ba097a80234997bff89ca113391c46 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "281228de85874ca1964dc443d6ef78ea" member_type: VOTER last_known_addr { host: "127.25.108.131" port: 32837 } } peers { permanent_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" member_type: VOTER last_known_addr { host: "127.25.108.129" port: 37639 } } peers { permanent_uuid: "a2ba097a80234997bff89ca113391c46" member_type: VOTER last_known_addr { host: "127.25.108.130" port: 43701 } }
09:04:08.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.848476 26076 catalog_manager.cc:5649] T 474f7edac15d4c468a54a651ead3a2a5 P a2ba097a80234997bff89ca113391c46 reported cstate change: term changed from 0 to 1, leader changed from <none> to a2ba097a80234997bff89ca113391c46 (127.25.108.130). New cstate: current_term: 1 leader_uuid: "a2ba097a80234997bff89ca113391c46" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "281228de85874ca1964dc443d6ef78ea" member_type: VOTER last_known_addr { host: "127.25.108.131" port: 32837 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" member_type: VOTER last_known_addr { host: "127.25.108.129" port: 37639 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a2ba097a80234997bff89ca113391c46" member_type: VOTER last_known_addr { host: "127.25.108.130" port: 43701 } health_report { overall_health: HEALTHY } } }
09:04:08.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.926625 26076 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:60516:
09:04:08.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
09:04:08.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
09:04:08.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:04:08.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
09:04:08.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
09:04:08.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:04:08.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:04:08.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:04:08.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:04:08.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:04:08.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:04:08.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:08.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:04:08.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
09:04:08.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
09:04:08.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:04:08.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:04:08.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:04:08.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:04:08.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:04:08.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:04:08.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:08.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:04:08.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
09:04:08.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
09:04:08.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:04:08.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:04:08.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:04:08.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:04:08.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:04:08.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:04:08.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:08.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:04:08.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
09:04:08.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
09:04:08.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
09:04:08.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:04:08.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:04:08.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:04:08.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:04:08.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:04:08.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:08.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:08.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
09:04:08.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
09:04:08.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
09:04:08.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
09:04:08.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:08.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
09:04:08.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
09:04:08.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:04:08.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
09:04:08.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:08.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:08.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:08.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
09:04:08.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:08.927596 26076 catalog_manager.cc:7011] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
09:04:08.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.941416 26562 tablet_service.cc:1505] Processing CreateTablet for tablet 2de22c9ead36457085cf570fcc66f08b (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=5ec6775cc92b49608f2878cc20b58f73]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
09:04:08.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.941761 26702 tablet_service.cc:1505] Processing CreateTablet for tablet 2de22c9ead36457085cf570fcc66f08b (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=5ec6775cc92b49608f2878cc20b58f73]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
09:04:08.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.942494 26702 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 2de22c9ead36457085cf570fcc66f08b. 1 dirs total, 0 dirs full, 0 dirs failed
09:04:08.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.942495 26562 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 2de22c9ead36457085cf570fcc66f08b. 1 dirs total, 0 dirs full, 0 dirs failed
09:04:08.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.948542 26419 tablet_service.cc:1505] Processing CreateTablet for tablet 2de22c9ead36457085cf570fcc66f08b (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=5ec6775cc92b49608f2878cc20b58f73]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
09:04:08.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.949424 26419 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 2de22c9ead36457085cf570fcc66f08b. 1 dirs total, 0 dirs full, 0 dirs failed
09:04:08.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.949885 26796 tablet_bootstrap.cc:492] T 2de22c9ead36457085cf570fcc66f08b P a2ba097a80234997bff89ca113391c46: Bootstrap starting.
09:04:08.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.949885 26795 tablet_bootstrap.cc:492] T 2de22c9ead36457085cf570fcc66f08b P 281228de85874ca1964dc443d6ef78ea: Bootstrap starting.
09:04:08.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.951754 26795 tablet_bootstrap.cc:654] T 2de22c9ead36457085cf570fcc66f08b P 281228de85874ca1964dc443d6ef78ea: Neither blocks nor log segments found. Creating new log.
09:04:08.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.951742 26796 tablet_bootstrap.cc:654] T 2de22c9ead36457085cf570fcc66f08b P a2ba097a80234997bff89ca113391c46: Neither blocks nor log segments found. Creating new log.
09:04:08.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.955288 26794 tablet_bootstrap.cc:492] T 2de22c9ead36457085cf570fcc66f08b P 06ecf9cc74fc4652a29bcab5f0d1d702: Bootstrap starting.
09:04:08.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.957013 26794 tablet_bootstrap.cc:654] T 2de22c9ead36457085cf570fcc66f08b P 06ecf9cc74fc4652a29bcab5f0d1d702: Neither blocks nor log segments found. Creating new log.
09:04:08.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.959493 26794 tablet_bootstrap.cc:492] T 2de22c9ead36457085cf570fcc66f08b P 06ecf9cc74fc4652a29bcab5f0d1d702: No bootstrap required, opened a new log
09:04:08.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.959677 26794 ts_tablet_manager.cc:1403] T 2de22c9ead36457085cf570fcc66f08b P 06ecf9cc74fc4652a29bcab5f0d1d702: Time spent bootstrapping tablet: real 0.004s user 0.001s sys 0.001s
09:04:08.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.960418 26794 raft_consensus.cc:359] T 2de22c9ead36457085cf570fcc66f08b P 06ecf9cc74fc4652a29bcab5f0d1d702 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" member_type: VOTER last_known_addr { host: "127.25.108.129" port: 37639 } } peers { permanent_uuid: "a2ba097a80234997bff89ca113391c46" member_type: VOTER last_known_addr { host: "127.25.108.130" port: 43701 } } peers { permanent_uuid: "281228de85874ca1964dc443d6ef78ea" member_type: VOTER last_known_addr { host: "127.25.108.131" port: 32837 } }
09:04:08.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.960608 26794 raft_consensus.cc:385] T 2de22c9ead36457085cf570fcc66f08b P 06ecf9cc74fc4652a29bcab5f0d1d702 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:04:08.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.960700 26794 raft_consensus.cc:740] T 2de22c9ead36457085cf570fcc66f08b P 06ecf9cc74fc4652a29bcab5f0d1d702 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 06ecf9cc74fc4652a29bcab5f0d1d702, State: Initialized, Role: FOLLOWER
09:04:08.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.960984 26794 consensus_queue.cc:260] T 2de22c9ead36457085cf570fcc66f08b P 06ecf9cc74fc4652a29bcab5f0d1d702 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" member_type: VOTER last_known_addr { host: "127.25.108.129" port: 37639 } } peers { permanent_uuid: "a2ba097a80234997bff89ca113391c46" member_type: VOTER last_known_addr { host: "127.25.108.130" port: 43701 } } peers { permanent_uuid: "281228de85874ca1964dc443d6ef78ea" member_type: VOTER last_known_addr { host: "127.25.108.131" port: 32837 } }
09:04:08.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.961504 26800 tablet_replica.cc:442] T 2de22c9ead36457085cf570fcc66f08b P 06ecf9cc74fc4652a29bcab5f0d1d702: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" member_type: VOTER last_known_addr { host: "127.25.108.129" port: 37639 } } peers { permanent_uuid: "a2ba097a80234997bff89ca113391c46" member_type: VOTER last_known_addr { host: "127.25.108.130" port: 43701 } } peers { permanent_uuid: "281228de85874ca1964dc443d6ef78ea" member_type: VOTER last_known_addr { host: "127.25.108.131" port: 32837 } } }
09:04:08.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.961743 26800 tablet_replica.cc:445] T 2de22c9ead36457085cf570fcc66f08b P 06ecf9cc74fc4652a29bcab5f0d1d702: This TxnStatusTablet replica's current role is: FOLLOWER
09:04:08.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.963828 26794 ts_tablet_manager.cc:1434] T 2de22c9ead36457085cf570fcc66f08b P 06ecf9cc74fc4652a29bcab5f0d1d702: Time spent starting tablet: real 0.004s user 0.000s sys 0.002s
09:04:08.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.965180 26795 tablet_bootstrap.cc:492] T 2de22c9ead36457085cf570fcc66f08b P 281228de85874ca1964dc443d6ef78ea: No bootstrap required, opened a new log
09:04:08.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.965406 26795 ts_tablet_manager.cc:1403] T 2de22c9ead36457085cf570fcc66f08b P 281228de85874ca1964dc443d6ef78ea: Time spent bootstrapping tablet: real 0.016s user 0.006s sys 0.003s
09:04:08.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.966782 26795 raft_consensus.cc:359] T 2de22c9ead36457085cf570fcc66f08b P 281228de85874ca1964dc443d6ef78ea [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" member_type: VOTER last_known_addr { host: "127.25.108.129" port: 37639 } } peers { permanent_uuid: "a2ba097a80234997bff89ca113391c46" member_type: VOTER last_known_addr { host: "127.25.108.130" port: 43701 } } peers { permanent_uuid: "281228de85874ca1964dc443d6ef78ea" member_type: VOTER last_known_addr { host: "127.25.108.131" port: 32837 } }
09:04:08.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.967084 26795 raft_consensus.cc:385] T 2de22c9ead36457085cf570fcc66f08b P 281228de85874ca1964dc443d6ef78ea [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:04:08.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.967180 26795 raft_consensus.cc:740] T 2de22c9ead36457085cf570fcc66f08b P 281228de85874ca1964dc443d6ef78ea [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 281228de85874ca1964dc443d6ef78ea, State: Initialized, Role: FOLLOWER
09:04:08.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.967473 26795 consensus_queue.cc:260] T 2de22c9ead36457085cf570fcc66f08b P 281228de85874ca1964dc443d6ef78ea [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" member_type: VOTER last_known_addr { host: "127.25.108.129" port: 37639 } } peers { permanent_uuid: "a2ba097a80234997bff89ca113391c46" member_type: VOTER last_known_addr { host: "127.25.108.130" port: 43701 } } peers { permanent_uuid: "281228de85874ca1964dc443d6ef78ea" member_type: VOTER last_known_addr { host: "127.25.108.131" port: 32837 } }
09:04:08.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.968143 26795 ts_tablet_manager.cc:1434] T 2de22c9ead36457085cf570fcc66f08b P 281228de85874ca1964dc443d6ef78ea: Time spent starting tablet: real 0.003s user 0.003s sys 0.000s
09:04:08.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.968186 26803 tablet_replica.cc:442] T 2de22c9ead36457085cf570fcc66f08b P 281228de85874ca1964dc443d6ef78ea: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" member_type: VOTER last_known_addr { host: "127.25.108.129" port: 37639 } } peers { permanent_uuid: "a2ba097a80234997bff89ca113391c46" member_type: VOTER last_known_addr { host: "127.25.108.130" port: 43701 } } peers { permanent_uuid: "281228de85874ca1964dc443d6ef78ea" member_type: VOTER last_known_addr { host: "127.25.108.131" port: 32837 } } }
09:04:08.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.968408 26803 tablet_replica.cc:445] T 2de22c9ead36457085cf570fcc66f08b P 281228de85874ca1964dc443d6ef78ea: This TxnStatusTablet replica's current role is: FOLLOWER
09:04:08.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.970001 26796 tablet_bootstrap.cc:492] T 2de22c9ead36457085cf570fcc66f08b P a2ba097a80234997bff89ca113391c46: No bootstrap required, opened a new log
09:04:08.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.970152 26796 ts_tablet_manager.cc:1403] T 2de22c9ead36457085cf570fcc66f08b P a2ba097a80234997bff89ca113391c46: Time spent bootstrapping tablet: real 0.020s user 0.008s sys 0.011s
09:04:08.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.970852 26796 raft_consensus.cc:359] T 2de22c9ead36457085cf570fcc66f08b P a2ba097a80234997bff89ca113391c46 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" member_type: VOTER last_known_addr { host: "127.25.108.129" port: 37639 } } peers { permanent_uuid: "a2ba097a80234997bff89ca113391c46" member_type: VOTER last_known_addr { host: "127.25.108.130" port: 43701 } } peers { permanent_uuid: "281228de85874ca1964dc443d6ef78ea" member_type: VOTER last_known_addr { host: "127.25.108.131" port: 32837 } }
09:04:08.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.971060 26796 raft_consensus.cc:385] T 2de22c9ead36457085cf570fcc66f08b P a2ba097a80234997bff89ca113391c46 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:04:08.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.971145 26796 raft_consensus.cc:740] T 2de22c9ead36457085cf570fcc66f08b P a2ba097a80234997bff89ca113391c46 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a2ba097a80234997bff89ca113391c46, State: Initialized, Role: FOLLOWER
09:04:08.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.971356 26796 consensus_queue.cc:260] T 2de22c9ead36457085cf570fcc66f08b P a2ba097a80234997bff89ca113391c46 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" member_type: VOTER last_known_addr { host: "127.25.108.129" port: 37639 } } peers { permanent_uuid: "a2ba097a80234997bff89ca113391c46" member_type: VOTER last_known_addr { host: "127.25.108.130" port: 43701 } } peers { permanent_uuid: "281228de85874ca1964dc443d6ef78ea" member_type: VOTER last_known_addr { host: "127.25.108.131" port: 32837 } }
09:04:08.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.971733 26796 ts_tablet_manager.cc:1434] T 2de22c9ead36457085cf570fcc66f08b P a2ba097a80234997bff89ca113391c46: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
09:04:08.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.972337 26801 tablet_replica.cc:442] T 2de22c9ead36457085cf570fcc66f08b P a2ba097a80234997bff89ca113391c46: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" member_type: VOTER last_known_addr { host: "127.25.108.129" port: 37639 } } peers { permanent_uuid: "a2ba097a80234997bff89ca113391c46" member_type: VOTER last_known_addr { host: "127.25.108.130" port: 43701 } } peers { permanent_uuid: "281228de85874ca1964dc443d6ef78ea" member_type: VOTER last_known_addr { host: "127.25.108.131" port: 32837 } } }
09:04:08.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:08.972571 26801 tablet_replica.cc:445] T 2de22c9ead36457085cf570fcc66f08b P a2ba097a80234997bff89ca113391c46: This TxnStatusTablet replica's current role is: FOLLOWER
09:04:09.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.016036 26800 raft_consensus.cc:493] T 2de22c9ead36457085cf570fcc66f08b P 06ecf9cc74fc4652a29bcab5f0d1d702 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
09:04:09.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.016250 26800 raft_consensus.cc:515] T 2de22c9ead36457085cf570fcc66f08b P 06ecf9cc74fc4652a29bcab5f0d1d702 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" member_type: VOTER last_known_addr { host: "127.25.108.129" port: 37639 } } peers { permanent_uuid: "a2ba097a80234997bff89ca113391c46" member_type: VOTER last_known_addr { host: "127.25.108.130" port: 43701 } } peers { permanent_uuid: "281228de85874ca1964dc443d6ef78ea" member_type: VOTER last_known_addr { host: "127.25.108.131" port: 32837 } }
09:04:09.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.016892 26800 leader_election.cc:290] T 2de22c9ead36457085cf570fcc66f08b P 06ecf9cc74fc4652a29bcab5f0d1d702 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a2ba097a80234997bff89ca113391c46 (127.25.108.130:43701), 281228de85874ca1964dc443d6ef78ea (127.25.108.131:32837)
09:04:09.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.017441 26582 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "2de22c9ead36457085cf570fcc66f08b" candidate_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a2ba097a80234997bff89ca113391c46" is_pre_election: true
09:04:09.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.017560 26723 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "2de22c9ead36457085cf570fcc66f08b" candidate_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "281228de85874ca1964dc443d6ef78ea" is_pre_election: true
09:04:09.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.017727 26582 raft_consensus.cc:2468] T 2de22c9ead36457085cf570fcc66f08b P a2ba097a80234997bff89ca113391c46 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 06ecf9cc74fc4652a29bcab5f0d1d702 in term 0.
09:04:09.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.017758 26723 raft_consensus.cc:2468] T 2de22c9ead36457085cf570fcc66f08b P 281228de85874ca1964dc443d6ef78ea [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 06ecf9cc74fc4652a29bcab5f0d1d702 in term 0.
09:04:09.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.018211 26363 leader_election.cc:304] T 2de22c9ead36457085cf570fcc66f08b P 06ecf9cc74fc4652a29bcab5f0d1d702 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 06ecf9cc74fc4652a29bcab5f0d1d702, a2ba097a80234997bff89ca113391c46; no voters:
09:04:09.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.018503 26800 raft_consensus.cc:2804] T 2de22c9ead36457085cf570fcc66f08b P 06ecf9cc74fc4652a29bcab5f0d1d702 [term 0 FOLLOWER]: Leader pre-election won for term 1
09:04:09.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.018603 26800 raft_consensus.cc:493] T 2de22c9ead36457085cf570fcc66f08b P 06ecf9cc74fc4652a29bcab5f0d1d702 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
09:04:09.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.018714 26800 raft_consensus.cc:3060] T 2de22c9ead36457085cf570fcc66f08b P 06ecf9cc74fc4652a29bcab5f0d1d702 [term 0 FOLLOWER]: Advancing to term 1
09:04:09.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.020727 26800 raft_consensus.cc:515] T 2de22c9ead36457085cf570fcc66f08b P 06ecf9cc74fc4652a29bcab5f0d1d702 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" member_type: VOTER last_known_addr { host: "127.25.108.129" port: 37639 } } peers { permanent_uuid: "a2ba097a80234997bff89ca113391c46" member_type: VOTER last_known_addr { host: "127.25.108.130" port: 43701 } } peers { permanent_uuid: "281228de85874ca1964dc443d6ef78ea" member_type: VOTER last_known_addr { host: "127.25.108.131" port: 32837 } }
09:04:09.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.021317 26800 leader_election.cc:290] T 2de22c9ead36457085cf570fcc66f08b P 06ecf9cc74fc4652a29bcab5f0d1d702 [CANDIDATE]: Term 1 election: Requested vote from peers a2ba097a80234997bff89ca113391c46 (127.25.108.130:43701), 281228de85874ca1964dc443d6ef78ea (127.25.108.131:32837)
09:04:09.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.021848 26582 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "2de22c9ead36457085cf570fcc66f08b" candidate_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a2ba097a80234997bff89ca113391c46"
09:04:09.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.021952 26723 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "2de22c9ead36457085cf570fcc66f08b" candidate_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "281228de85874ca1964dc443d6ef78ea"
09:04:09.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.022089 26582 raft_consensus.cc:3060] T 2de22c9ead36457085cf570fcc66f08b P a2ba097a80234997bff89ca113391c46 [term 0 FOLLOWER]: Advancing to term 1
09:04:09.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.022118 26723 raft_consensus.cc:3060] T 2de22c9ead36457085cf570fcc66f08b P 281228de85874ca1964dc443d6ef78ea [term 0 FOLLOWER]: Advancing to term 1
09:04:09.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.024065 26582 raft_consensus.cc:2468] T 2de22c9ead36457085cf570fcc66f08b P a2ba097a80234997bff89ca113391c46 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 06ecf9cc74fc4652a29bcab5f0d1d702 in term 1.
09:04:09.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.024087 26723 raft_consensus.cc:2468] T 2de22c9ead36457085cf570fcc66f08b P 281228de85874ca1964dc443d6ef78ea [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 06ecf9cc74fc4652a29bcab5f0d1d702 in term 1.
09:04:09.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.024470 26363 leader_election.cc:304] T 2de22c9ead36457085cf570fcc66f08b P 06ecf9cc74fc4652a29bcab5f0d1d702 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 06ecf9cc74fc4652a29bcab5f0d1d702, 281228de85874ca1964dc443d6ef78ea; no voters:
09:04:09.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.024782 26800 raft_consensus.cc:2804] T 2de22c9ead36457085cf570fcc66f08b P 06ecf9cc74fc4652a29bcab5f0d1d702 [term 1 FOLLOWER]: Leader election won for term 1
09:04:09.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.024937 26800 raft_consensus.cc:697] T 2de22c9ead36457085cf570fcc66f08b P 06ecf9cc74fc4652a29bcab5f0d1d702 [term 1 LEADER]: Becoming Leader. State: Replica: 06ecf9cc74fc4652a29bcab5f0d1d702, State: Running, Role: LEADER
09:04:09.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.025211 26800 consensus_queue.cc:237] T 2de22c9ead36457085cf570fcc66f08b P 06ecf9cc74fc4652a29bcab5f0d1d702 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" member_type: VOTER last_known_addr { host: "127.25.108.129" port: 37639 } } peers { permanent_uuid: "a2ba097a80234997bff89ca113391c46" member_type: VOTER last_known_addr { host: "127.25.108.130" port: 43701 } } peers { permanent_uuid: "281228de85874ca1964dc443d6ef78ea" member_type: VOTER last_known_addr { host: "127.25.108.131" port: 32837 } }
09:04:09.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.027457 26808 tablet_replica.cc:442] T 2de22c9ead36457085cf570fcc66f08b P 06ecf9cc74fc4652a29bcab5f0d1d702: TxnStatusTablet state changed. Reason: New leader 06ecf9cc74fc4652a29bcab5f0d1d702. Latest consensus state: current_term: 1 leader_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" member_type: VOTER last_known_addr { host: "127.25.108.129" port: 37639 } } peers { permanent_uuid: "a2ba097a80234997bff89ca113391c46" member_type: VOTER last_known_addr { host: "127.25.108.130" port: 43701 } } peers { permanent_uuid: "281228de85874ca1964dc443d6ef78ea" member_type: VOTER last_known_addr { host: "127.25.108.131" port: 32837 } } }
09:04:09.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.027787 26808 tablet_replica.cc:445] T 2de22c9ead36457085cf570fcc66f08b P 06ecf9cc74fc4652a29bcab5f0d1d702: This TxnStatusTablet replica's current role is: LEADER
09:04:09.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.028832 26074 catalog_manager.cc:5649] T 2de22c9ead36457085cf570fcc66f08b P 06ecf9cc74fc4652a29bcab5f0d1d702 reported cstate change: term changed from 0 to 1, leader changed from <none> to 06ecf9cc74fc4652a29bcab5f0d1d702 (127.25.108.129). New cstate: current_term: 1 leader_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" member_type: VOTER last_known_addr { host: "127.25.108.129" port: 37639 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a2ba097a80234997bff89ca113391c46" member_type: VOTER last_known_addr { host: "127.25.108.130" port: 43701 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "281228de85874ca1964dc443d6ef78ea" member_type: VOTER last_known_addr { host: "127.25.108.131" port: 32837 } health_report { overall_health: UNKNOWN } } }
09:04:09.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.040395 26827 txn_status_manager.cc:874] Waiting until node catch up with all replicated operations in previous term...
09:04:09.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.040758 26827 txn_status_manager.cc:930] Loading transaction status metadata into memory...
09:04:09.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.072278 26723 raft_consensus.cc:1275] T 2de22c9ead36457085cf570fcc66f08b P 281228de85874ca1964dc443d6ef78ea [term 1 FOLLOWER]: Refusing update from remote peer 06ecf9cc74fc4652a29bcab5f0d1d702: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
09:04:09.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.072388 26582 raft_consensus.cc:1275] T 2de22c9ead36457085cf570fcc66f08b P a2ba097a80234997bff89ca113391c46 [term 1 FOLLOWER]: Refusing update from remote peer 06ecf9cc74fc4652a29bcab5f0d1d702: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
09:04:09.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.073282 26808 consensus_queue.cc:1048] T 2de22c9ead36457085cf570fcc66f08b P 06ecf9cc74fc4652a29bcab5f0d1d702 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a2ba097a80234997bff89ca113391c46" member_type: VOTER last_known_addr { host: "127.25.108.130" port: 43701 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
09:04:09.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.073572 26800 consensus_queue.cc:1048] T 2de22c9ead36457085cf570fcc66f08b P 06ecf9cc74fc4652a29bcab5f0d1d702 [LEADER]: Connected to new peer: Peer: permanent_uuid: "281228de85874ca1964dc443d6ef78ea" member_type: VOTER last_known_addr { host: "127.25.108.131" port: 32837 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
09:04:09.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.082585 26803 tablet_replica.cc:442] T 2de22c9ead36457085cf570fcc66f08b P 281228de85874ca1964dc443d6ef78ea: TxnStatusTablet state changed. Reason: New leader 06ecf9cc74fc4652a29bcab5f0d1d702. Latest consensus state: current_term: 1 leader_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" member_type: VOTER last_known_addr { host: "127.25.108.129" port: 37639 } } peers { permanent_uuid: "a2ba097a80234997bff89ca113391c46" member_type: VOTER last_known_addr { host: "127.25.108.130" port: 43701 } } peers { permanent_uuid: "281228de85874ca1964dc443d6ef78ea" member_type: VOTER last_known_addr { host: "127.25.108.131" port: 32837 } } }
09:04:09.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.082811 26803 tablet_replica.cc:445] T 2de22c9ead36457085cf570fcc66f08b P 281228de85874ca1964dc443d6ef78ea: This TxnStatusTablet replica's current role is: FOLLOWER
09:04:09.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.085884 26808 tablet_replica.cc:442] T 2de22c9ead36457085cf570fcc66f08b P 06ecf9cc74fc4652a29bcab5f0d1d702: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" member_type: VOTER last_known_addr { host: "127.25.108.129" port: 37639 } } peers { permanent_uuid: "a2ba097a80234997bff89ca113391c46" member_type: VOTER last_known_addr { host: "127.25.108.130" port: 43701 } } peers { permanent_uuid: "281228de85874ca1964dc443d6ef78ea" member_type: VOTER last_known_addr { host: "127.25.108.131" port: 32837 } } }
09:04:09.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.086184 26808 tablet_replica.cc:445] T 2de22c9ead36457085cf570fcc66f08b P 06ecf9cc74fc4652a29bcab5f0d1d702: This TxnStatusTablet replica's current role is: LEADER
09:04:09.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.087632 26831 mvcc.cc:204] Tried to move back new op lower bound from 7216040546589528064 to 7216040546409508864. Current Snapshot: MvccSnapshot[applied={T|T < 7216040546589528064}]
09:04:09.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.097558 26801 tablet_replica.cc:442] T 2de22c9ead36457085cf570fcc66f08b P a2ba097a80234997bff89ca113391c46: TxnStatusTablet state changed. Reason: New leader 06ecf9cc74fc4652a29bcab5f0d1d702. Latest consensus state: current_term: 1 leader_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" member_type: VOTER last_known_addr { host: "127.25.108.129" port: 37639 } } peers { permanent_uuid: "a2ba097a80234997bff89ca113391c46" member_type: VOTER last_known_addr { host: "127.25.108.130" port: 43701 } } peers { permanent_uuid: "281228de85874ca1964dc443d6ef78ea" member_type: VOTER last_known_addr { host: "127.25.108.131" port: 32837 } } }
09:04:09.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.097853 26801 tablet_replica.cc:445] T 2de22c9ead36457085cf570fcc66f08b P a2ba097a80234997bff89ca113391c46: This TxnStatusTablet replica's current role is: FOLLOWER
09:04:09.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.100535 26800 tablet_replica.cc:442] T 2de22c9ead36457085cf570fcc66f08b P 06ecf9cc74fc4652a29bcab5f0d1d702: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" member_type: VOTER last_known_addr { host: "127.25.108.129" port: 37639 } } peers { permanent_uuid: "a2ba097a80234997bff89ca113391c46" member_type: VOTER last_known_addr { host: "127.25.108.130" port: 43701 } } peers { permanent_uuid: "281228de85874ca1964dc443d6ef78ea" member_type: VOTER last_known_addr { host: "127.25.108.131" port: 32837 } } }
09:04:09.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.100878 26800 tablet_replica.cc:445] T 2de22c9ead36457085cf570fcc66f08b P 06ecf9cc74fc4652a29bcab5f0d1d702: This TxnStatusTablet replica's current role is: LEADER
09:04:09.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.104570 26803 tablet_replica.cc:442] T 2de22c9ead36457085cf570fcc66f08b P 281228de85874ca1964dc443d6ef78ea: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" member_type: VOTER last_known_addr { host: "127.25.108.129" port: 37639 } } peers { permanent_uuid: "a2ba097a80234997bff89ca113391c46" member_type: VOTER last_known_addr { host: "127.25.108.130" port: 43701 } } peers { permanent_uuid: "281228de85874ca1964dc443d6ef78ea" member_type: VOTER last_known_addr { host: "127.25.108.131" port: 32837 } } }
09:04:09.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.104781 26803 tablet_replica.cc:445] T 2de22c9ead36457085cf570fcc66f08b P 281228de85874ca1964dc443d6ef78ea: This TxnStatusTablet replica's current role is: FOLLOWER
09:04:09.108 [DEBUG - Time-limited test] (KuduTransaction.java:702) starting keepalive heartbeating with period 500 ms (txn ID 0)
09:04:09.111 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 1461c31339b24f7783bf16fa929cf842
09:04:09.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.112444 26801 tablet_replica.cc:442] T 2de22c9ead36457085cf570fcc66f08b P a2ba097a80234997bff89ca113391c46: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" member_type: VOTER last_known_addr { host: "127.25.108.129" port: 37639 } } peers { permanent_uuid: "a2ba097a80234997bff89ca113391c46" member_type: VOTER last_known_addr { host: "127.25.108.130" port: 43701 } } peers { permanent_uuid: "281228de85874ca1964dc443d6ef78ea" member_type: VOTER last_known_addr { host: "127.25.108.131" port: 32837 } } }
09:04:09.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.112692 26801 tablet_replica.cc:445] T 2de22c9ead36457085cf570fcc66f08b P a2ba097a80234997bff89ca113391c46: This TxnStatusTablet replica's current role is: FOLLOWER
09:04:09.113 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 474f7edac15d4c468a54a651ead3a2a5 for table 'txn_manager_fallback' with partition [0x00000000, 0x00000001)
09:04:09.114 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8bf23027b1c64e248061f4f4a34dc7a6 for table 'txn_manager_fallback' with partition [0x00000001, 0x00000002)
09:04:09.114 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x00000000, upperBoundPartitionKey=0x00000001, ttl=299999, tablet=474f7edac15d4c468a54a651ead3a2a5@[06ecf9cc74fc4652a29bcab5f0d1d702(127.25.108.129:37639),281228de85874ca1964dc443d6ef78ea(127.25.108.131:32837),a2ba097a80234997bff89ca113391c46(127.25.108.130:43701)[L]]}, Tablet{lowerBoundPartitionKey=0x00000001, upperBoundPartitionKey=0x00000002, ttl=299999, tablet=8bf23027b1c64e248061f4f4a34dc7a6@[06ecf9cc74fc4652a29bcab5f0d1d702(127.25.108.129:37639)[L],281228de85874ca1964dc443d6ef78ea(127.25.108.131:32837),a2ba097a80234997bff89ca113391c46(127.25.108.130:43701)]}, NonCoveredRange{lowerBoundPartitionKey=0x00000002, upperBoundPartitionKey=0x, ttl=299999}]
09:04:09.116 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=4), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC GetTableLocations: sending RPC to server master-127.25.108.190:40189, [2ms] Sub RPC GetTableLocations: received response from server master-127.25.108.190:40189: OK, deferred=null) after lookup
09:04:09.116 [DEBUG - kudu-nio-0] (Connection.java:748) [peer a2ba097a80234997bff89ca113391c46(127.25.108.130:43701)] connecting to peer
09:04:09.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.117615 26800 consensus_queue.cc:1048] T 8bf23027b1c64e248061f4f4a34dc7a6 P 06ecf9cc74fc4652a29bcab5f0d1d702 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a2ba097a80234997bff89ca113391c46" member_type: VOTER last_known_addr { host: "127.25.108.130" port: 43701 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
09:04:09.118 [DEBUG - kudu-nio-1] (Connection.java:757) [peer a2ba097a80234997bff89ca113391c46(127.25.108.130:43701)] Successfully connected to peer
09:04:09.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.127017 26800 consensus_queue.cc:1048] T 8bf23027b1c64e248061f4f4a34dc7a6 P 06ecf9cc74fc4652a29bcab5f0d1d702 [LEADER]: Connected to new peer: Peer: permanent_uuid: "281228de85874ca1964dc443d6ef78ea" member_type: VOTER last_known_addr { host: "127.25.108.131" port: 32837 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
09:04:09.136 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x8f7ec695, L:/127.0.0.1:52828 - R:/127.25.108.130:43701] using TOKEN/null
09:04:09.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.167621 26722 raft_consensus.cc:1275] T 474f7edac15d4c468a54a651ead3a2a5 P 281228de85874ca1964dc443d6ef78ea [term 1 FOLLOWER]: Refusing update from remote peer a2ba097a80234997bff89ca113391c46: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
09:04:09.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.167943 26439 raft_consensus.cc:1275] T 474f7edac15d4c468a54a651ead3a2a5 P 06ecf9cc74fc4652a29bcab5f0d1d702 [term 1 FOLLOWER]: Refusing update from remote peer a2ba097a80234997bff89ca113391c46: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
09:04:09.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.168331 26813 consensus_queue.cc:1048] T 474f7edac15d4c468a54a651ead3a2a5 P a2ba097a80234997bff89ca113391c46 [LEADER]: Connected to new peer: Peer: permanent_uuid: "281228de85874ca1964dc443d6ef78ea" member_type: VOTER last_known_addr { host: "127.25.108.131" port: 32837 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
09:04:09.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.168632 26801 consensus_queue.cc:1048] T 474f7edac15d4c468a54a651ead3a2a5 P a2ba097a80234997bff89ca113391c46 [LEADER]: Connected to new peer: Peer: permanent_uuid: "06ecf9cc74fc4652a29bcab5f0d1d702" member_type: VOTER last_known_addr { host: "127.25.108.129" port: 37639 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
09:04:09.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.178002 26833 mvcc.cc:204] Tried to move back new op lower bound from 7216040546984505344 to 7216040545664786432. Current Snapshot: MvccSnapshot[applied={T|T < 7216040546984505344}]
09:04:09.190 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.25.108.190:40189
09:04:09.191 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
09:04:09.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.191857 26034 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskescuMK/build/asan/bin/kudu pid 26045
09:04:09.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.193404 26116 generic_service.cc:196] Checking for leaks (request via RPC)
09:04:09.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.329414 26034 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskescuMK/build/asan/bin/kudu with pid 26045
09:04:09.360 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.25.108.190:40189(127.25.108.190:40189)] handling channelInactive
09:04:09.360 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.108.190:40189(127.25.108.190:40189)] cleaning up while in state READY due to: connection closed
09:04:09.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:09.361003 26628 heartbeater.cc:646] Failed to heartbeat to 127.25.108.190:40189 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv got EOF from 127.25.108.190:40189 (error 108)
09:04:09.361 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:04:09.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:09.549439 26485 heartbeater.cc:646] Failed to heartbeat to 127.25.108.190:40189 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.25.108.190:40189: connect: Connection refused (error 111)
09:04:09.628 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.25.108.190:40189(127.25.108.190:40189)] connecting to peer
09:04:09.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.629303 26849 raft_consensus.cc:493] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader aaeb9a61bdca4ab8bdc0ee318551ab52)
09:04:09.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.629525 26850 raft_consensus.cc:493] T 00000000000000000000000000000000 P db0c22fe226d4071b623e4f6921f4a3d [term 1 FOLLOWER]: Starting pre-election (detected failure of leader aaeb9a61bdca4ab8bdc0ee318551ab52)
09:04:09.629 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189
09:04:09.629 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.108.190:40189(127.25.108.190:40189)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189
09:04:09.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.629734 26849 raft_consensus.cc:515] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aaeb9a61bdca4ab8bdc0ee318551ab52" member_type: VOTER last_known_addr { host: "127.25.108.190" port: 40189 } } peers { permanent_uuid: "0ec5572f7e8c4d27bb850d5983c8eb07" member_type: VOTER last_known_addr { host: "127.25.108.189" port: 37911 } } peers { permanent_uuid: "db0c22fe226d4071b623e4f6921f4a3d" member_type: VOTER last_known_addr { host: "127.25.108.188" port: 38177 } }
09:04:09.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.629858 26850 raft_consensus.cc:515] T 00000000000000000000000000000000 P db0c22fe226d4071b623e4f6921f4a3d [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aaeb9a61bdca4ab8bdc0ee318551ab52" member_type: VOTER last_known_addr { host: "127.25.108.190" port: 40189 } } peers { permanent_uuid: "0ec5572f7e8c4d27bb850d5983c8eb07" member_type: VOTER last_known_addr { host: "127.25.108.189" port: 37911 } } peers { permanent_uuid: "db0c22fe226d4071b623e4f6921f4a3d" member_type: VOTER last_known_addr { host: "127.25.108.188" port: 38177 } }
09:04:09.630 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location master-127.25.108.190:40189(127.25.108.190:40189) for tablet Kudu Master: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189
09:04:09.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.631290 26849 leader_election.cc:290] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers aaeb9a61bdca4ab8bdc0ee318551ab52 (127.25.108.190:40189), db0c22fe226d4071b623e4f6921f4a3d (127.25.108.188:38177)
09:04:09.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.631426 26850 leader_election.cc:290] T 00000000000000000000000000000000 P db0c22fe226d4071b623e4f6921f4a3d [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers aaeb9a61bdca4ab8bdc0ee318551ab52 (127.25.108.190:40189), 0ec5572f7e8c4d27bb850d5983c8eb07 (127.25.108.189:37911)
09:04:09.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.632269 26165 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "db0c22fe226d4071b623e4f6921f4a3d" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "0ec5572f7e8c4d27bb850d5983c8eb07" is_pre_election: true
09:04:09.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.632309 26247 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0ec5572f7e8c4d27bb850d5983c8eb07" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "db0c22fe226d4071b623e4f6921f4a3d" is_pre_election: true
09:04:09.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.632512 26165 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate db0c22fe226d4071b623e4f6921f4a3d in term 1.
09:04:09.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.632514 26247 raft_consensus.cc:2468] T 00000000000000000000000000000000 P db0c22fe226d4071b623e4f6921f4a3d [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0ec5572f7e8c4d27bb850d5983c8eb07 in term 1.
09:04:09.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:09.632473 26225 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.25.108.190:40189: connect: Connection refused (error 111)
09:04:09.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.633036 26143 leader_election.cc:304] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0ec5572f7e8c4d27bb850d5983c8eb07, db0c22fe226d4071b623e4f6921f4a3d; no voters:
09:04:09.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.633333 26223 leader_election.cc:304] T 00000000000000000000000000000000 P db0c22fe226d4071b623e4f6921f4a3d [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0ec5572f7e8c4d27bb850d5983c8eb07, db0c22fe226d4071b623e4f6921f4a3d; no voters:
09:04:09.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.633534 26849 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07 [term 1 FOLLOWER]: Leader pre-election won for term 2
09:04:09.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.633672 26849 raft_consensus.cc:493] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07 [term 1 FOLLOWER]: Starting leader election (detected failure of leader aaeb9a61bdca4ab8bdc0ee318551ab52)
09:04:09.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.633687 26850 raft_consensus.cc:2804] T 00000000000000000000000000000000 P db0c22fe226d4071b623e4f6921f4a3d [term 1 FOLLOWER]: Leader pre-election won for term 2
09:04:09.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.633781 26849 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07 [term 1 FOLLOWER]: Advancing to term 2
09:04:09.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.633795 26850 raft_consensus.cc:493] T 00000000000000000000000000000000 P db0c22fe226d4071b623e4f6921f4a3d [term 1 FOLLOWER]: Starting leader election (detected failure of leader aaeb9a61bdca4ab8bdc0ee318551ab52)
09:04:09.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.633898 26850 raft_consensus.cc:3060] T 00000000000000000000000000000000 P db0c22fe226d4071b623e4f6921f4a3d [term 1 FOLLOWER]: Advancing to term 2
09:04:09.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:09.634624 26225 leader_election.cc:336] T 00000000000000000000000000000000 P db0c22fe226d4071b623e4f6921f4a3d [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer aaeb9a61bdca4ab8bdc0ee318551ab52 (127.25.108.190:40189): Network error: Client connection negotiation failed: client connection to 127.25.108.190:40189: connect: Connection refused (error 111)
09:04:09.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:09.636608 26144 leader_election.cc:336] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer aaeb9a61bdca4ab8bdc0ee318551ab52 (127.25.108.190:40189): Network error: Client connection negotiation failed: client connection to 127.25.108.190:40189: connect: Connection refused (error 111)
09:04:09.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.636850 26850 raft_consensus.cc:515] T 00000000000000000000000000000000 P db0c22fe226d4071b623e4f6921f4a3d [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aaeb9a61bdca4ab8bdc0ee318551ab52" member_type: VOTER last_known_addr { host: "127.25.108.190" port: 40189 } } peers { permanent_uuid: "0ec5572f7e8c4d27bb850d5983c8eb07" member_type: VOTER last_known_addr { host: "127.25.108.189" port: 37911 } } peers { permanent_uuid: "db0c22fe226d4071b623e4f6921f4a3d" member_type: VOTER last_known_addr { host: "127.25.108.188" port: 38177 } }
09:04:09.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.637152 26849 raft_consensus.cc:515] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aaeb9a61bdca4ab8bdc0ee318551ab52" member_type: VOTER last_known_addr { host: "127.25.108.190" port: 40189 } } peers { permanent_uuid: "0ec5572f7e8c4d27bb850d5983c8eb07" member_type: VOTER last_known_addr { host: "127.25.108.189" port: 37911 } } peers { permanent_uuid: "db0c22fe226d4071b623e4f6921f4a3d" member_type: VOTER last_known_addr { host: "127.25.108.188" port: 38177 } }
09:04:09.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.637476 26850 leader_election.cc:290] T 00000000000000000000000000000000 P db0c22fe226d4071b623e4f6921f4a3d [CANDIDATE]: Term 2 election: Requested vote from peers aaeb9a61bdca4ab8bdc0ee318551ab52 (127.25.108.190:40189), 0ec5572f7e8c4d27bb850d5983c8eb07 (127.25.108.189:37911)
09:04:09.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.637744 26849 leader_election.cc:290] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07 [CANDIDATE]: Term 2 election: Requested vote from peers aaeb9a61bdca4ab8bdc0ee318551ab52 (127.25.108.190:40189), db0c22fe226d4071b623e4f6921f4a3d (127.25.108.188:38177)
09:04:09.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.638000 26165 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "db0c22fe226d4071b623e4f6921f4a3d" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "0ec5572f7e8c4d27bb850d5983c8eb07"
09:04:09.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.638267 26165 raft_consensus.cc:2393] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate db0c22fe226d4071b623e4f6921f4a3d in current term 2: Already voted for candidate 0ec5572f7e8c4d27bb850d5983c8eb07 in this term.
09:04:09.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.638262 26247 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0ec5572f7e8c4d27bb850d5983c8eb07" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "db0c22fe226d4071b623e4f6921f4a3d"
09:04:09.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.638509 26247 raft_consensus.cc:2393] T 00000000000000000000000000000000 P db0c22fe226d4071b623e4f6921f4a3d [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 0ec5572f7e8c4d27bb850d5983c8eb07 in current term 2: Already voted for candidate db0c22fe226d4071b623e4f6921f4a3d in this term.
09:04:09.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:09.638832 26225 leader_election.cc:336] T 00000000000000000000000000000000 P db0c22fe226d4071b623e4f6921f4a3d [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer aaeb9a61bdca4ab8bdc0ee318551ab52 (127.25.108.190:40189): Network error: Client connection negotiation failed: client connection to 127.25.108.190:40189: connect: Connection refused (error 111)
09:04:09.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.638971 26225 leader_election.cc:304] T 00000000000000000000000000000000 P db0c22fe226d4071b623e4f6921f4a3d [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: db0c22fe226d4071b623e4f6921f4a3d; no voters: 0ec5572f7e8c4d27bb850d5983c8eb07, aaeb9a61bdca4ab8bdc0ee318551ab52
09:04:09.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.639281 26850 raft_consensus.cc:2749] T 00000000000000000000000000000000 P db0c22fe226d4071b623e4f6921f4a3d [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
09:04:09.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:09.639530 26144 leader_election.cc:336] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer aaeb9a61bdca4ab8bdc0ee318551ab52 (127.25.108.190:40189): Network error: Client connection negotiation failed: client connection to 127.25.108.190:40189: connect: Connection refused (error 111)
09:04:09.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.639657 26144 leader_election.cc:304] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 0ec5572f7e8c4d27bb850d5983c8eb07; no voters: aaeb9a61bdca4ab8bdc0ee318551ab52, db0c22fe226d4071b623e4f6921f4a3d
09:04:09.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:09.639937 26849 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
09:04:09.645 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.25.108.190:40189(127.25.108.190:40189)] connecting to peer
09:04:09.646 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189
09:04:09.646 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.108.190:40189(127.25.108.190:40189)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189
09:04:09.646 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.108.190:40189: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189
09:04:09.647 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.108.190:40189,127.25.108.188:38177,127.25.108.189:37911; will retry
09:04:09.665 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.25.108.190:40189(127.25.108.190:40189)] connecting to peer
09:04:09.666 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189
09:04:09.666 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.108.190:40189(127.25.108.190:40189)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189
09:04:09.667 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.108.190:40189: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189
09:04:09.667 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.108.190:40189,127.25.108.188:38177,127.25.108.189:37911; will retry
09:04:09.685 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.25.108.190:40189(127.25.108.190:40189)] connecting to peer
09:04:09.686 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189
09:04:09.686 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.108.190:40189(127.25.108.190:40189)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189
09:04:09.686 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.108.190:40189: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189
09:04:09.687 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.108.190:40189,127.25.108.188:38177,127.25.108.189:37911; will retry
09:04:09.705 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.25.108.190:40189(127.25.108.190:40189)] connecting to peer
09:04:09.707 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189
09:04:09.707 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.108.190:40189(127.25.108.190:40189)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189
09:04:09.708 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.108.190:40189: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189
09:04:09.709 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.108.190:40189,127.25.108.188:38177,127.25.108.189:37911; will retry
09:04:09.725 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.25.108.190:40189(127.25.108.190:40189)] connecting to peer
09:04:09.726 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189
09:04:09.726 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.108.190:40189(127.25.108.190:40189)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189
09:04:09.726 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.108.190:40189: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189
09:04:09.727 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.108.190:40189,127.25.108.188:38177,127.25.108.189:37911; will retry
09:04:09.785 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.25.108.190:40189(127.25.108.190:40189)] connecting to peer
09:04:09.786 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189
09:04:09.786 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.108.190:40189(127.25.108.190:40189)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189
09:04:09.786 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.108.190:40189: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189
09:04:09.787 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.108.190:40189,127.25.108.188:38177,127.25.108.189:37911; will retry
09:04:09.788 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=7, TimeoutTracker(timeout=250, elapsed=163), Traces: [0ms] sending RPC to server master-127.25.108.190:40189, [5ms] received response from server master-127.25.108.190:40189: Network error: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189, [5ms] delaying RPC due to: Network error: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.190:40189, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.188:38177, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.189:37911, [21ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.190:40189: Network error: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189, [22ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.189:37911: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.188:38177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.25.108.190:40189,127.25.108.188:38177,127.25.108.189:37911) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.190:40189, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.188:38177, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.189:37911, [41ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.190:40189: Network error: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189, [42ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.189:37911: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.188:38177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.25.108.190:40189,127.25.108.188:38177,127.25.108.189:37911) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.190:40189, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.188:38177, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.189:37911, [61ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.190:40189: Network error: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189, [62ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.188:38177: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.189:37911: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.25.108.190:40189,127.25.108.188:38177,127.25.108.189:37911) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.190:40189, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.188:38177, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.189:37911, [82ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.188:38177: OK, [83ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.190:40189: Network error: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189, [84ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.189:37911: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.25.108.190:40189,127.25.108.188:38177,127.25.108.189:37911) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189, [99ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.190:40189, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.188:38177, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.189:37911, [101ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.190:40189: Network error: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189, [102ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.188:38177: OK, [102ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.189:37911: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.25.108.190:40189,127.25.108.188:38177,127.25.108.189:37911) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.190:40189, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.188:38177, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.189:37911, [161ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.190:40189: Network error: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189, [162ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.188:38177: OK, [162ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.189:37911: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.25.108.190:40189,127.25.108.188:38177,127.25.108.189:37911) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189, deferred=Deferred@1713682084(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$117/1797215806@2625e62c -> (continuation of Deferred@2088764686 after retry RPC after error@1774938469) -> (continuation of Deferred@714699630 after retry RPC after error@595911831) -> (continuation of Deferred@1599094172 after retry RPC after error@1835202386) -> (continuation of Deferred@209841043 after retry RPC after error@136648392) -> (continuation of Deferred@1447523981 after retry RPC after error@744870768), errback=org.apache.kudu.client.KuduTransaction$$Lambda$118/2034592728@2fbb781c -> (continuation of Deferred@2088764686 after retry RPC after error@1774938469) -> (continuation of Deferred@714699630 after retry RPC after error@595911831) -> (continuation of Deferred@1599094172 after retry RPC after error@1835202386) -> (continuation of Deferred@209841043 after retry RPC after error@136648392) -> (continuation of Deferred@1447523981 after retry RPC after error@744870768)))
09:04:09.789 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=7, TimeoutTracker(timeout=250, elapsed=163), Traces: [0ms] sending RPC to server master-127.25.108.190:40189, [5ms] received response from server master-127.25.108.190:40189: Network error: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189, [5ms] delaying RPC due to: Network error: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.190:40189, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.188:38177, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.189:37911, [21ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.190:40189: Network error: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189, [22ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.189:37911: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.188:38177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.25.108.190:40189,127.25.108.188:38177,127.25.108.189:37911) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.190:40189, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.188:38177, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.189:37911, [41ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.190:40189: Network error: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189, [42ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.189:37911: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.188:38177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.25.108.190:40189,127.25.108.188:38177,127.25.108.189:37911) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.190:40189, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.188:38177, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.189:37911, [61ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.190:40189: Network error: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189, [62ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.188:38177: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.189:37911: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.25.108.190:40189,127.25.108.188:38177,127.25.108.189:37911) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.190:40189, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.188:38177, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.189:37911, [82ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.188:38177: OK, [83ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.190:40189: Network error: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189, [84ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.189:37911: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.25.108.190:40189,127.25.108.188:38177,127.25.108.189:37911) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189, [99ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.190:40189, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.188:38177, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.189:37911, [101ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.190:40189: Network error: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189, [102ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.188:38177: OK, [102ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.189:37911: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.25.108.190:40189,127.25.108.188:38177,127.25.108.189:37911) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.190:40189, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.188:38177, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.189:37911, [161ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.190:40189: Network error: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189, [162ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.188:38177: OK, [162ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.189:37911: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.25.108.190:40189,127.25.108.188:38177,127.25.108.189:37911) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189, deferred=Deferred@1713682084(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$117/1797215806@2625e62c -> (continuation of Deferred@2088764686 after retry RPC after error@1774938469) -> (continuation of Deferred@714699630 after retry RPC after error@595911831) -> (continuation of Deferred@1599094172 after retry RPC after error@1835202386) -> (continuation of Deferred@209841043 after retry RPC after error@136648392) -> (continuation of Deferred@1447523981 after retry RPC after error@744870768), errback=org.apache.kudu.client.KuduTransaction$$Lambda$118/2034592728@2fbb781c -> (continuation of Deferred@2088764686 after retry RPC after error@1774938469) -> (continuation of Deferred@714699630 after retry RPC after error@595911831) -> (continuation of Deferred@1599094172 after retry RPC after error@1835202386) -> (continuation of Deferred@209841043 after retry RPC after error@136648392) -> (continuation of Deferred@1447523981 after retry RPC after error@744870768)))
09:04:09.805 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.25.108.190:40189(127.25.108.190:40189)] connecting to peer
09:04:09.806 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189
09:04:09.806 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.108.190:40189(127.25.108.190:40189)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189
09:04:09.806 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.108.190:40189: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189
09:04:09.807 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.108.190:40189,127.25.108.188:38177,127.25.108.189:37911; will retry
09:04:09.825 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.25.108.190:40189(127.25.108.190:40189)] connecting to peer
09:04:09.826 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189
09:04:09.826 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.108.190:40189(127.25.108.190:40189)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189
09:04:09.827 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.108.190:40189: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189
09:04:09.827 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.108.190:40189,127.25.108.188:38177,127.25.108.189:37911; will retry
09:04:09.845 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.25.108.190:40189(127.25.108.190:40189)] connecting to peer
09:04:09.847 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189
09:04:09.847 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.108.190:40189(127.25.108.190:40189)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189
09:04:09.847 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.108.190:40189: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189
09:04:09.848 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.108.190:40189,127.25.108.188:38177,127.25.108.189:37911; will retry
09:04:09.864 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.25.108.190:40189(127.25.108.190:40189)] connecting to peer
09:04:09.866 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189
09:04:09.866 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.108.190:40189(127.25.108.190:40189)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189
09:04:09.866 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.108.190:40189: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189
09:04:09.867 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.108.190:40189,127.25.108.188:38177,127.25.108.189:37911; will retry
09:04:09.884 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.25.108.190:40189(127.25.108.190:40189)] connecting to peer
09:04:09.886 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189
09:04:09.886 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.108.190:40189(127.25.108.190:40189)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189
09:04:09.886 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.108.190:40189: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189
09:04:09.887 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.108.190:40189,127.25.108.188:38177,127.25.108.189:37911; will retry
09:04:09.924 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.25.108.190:40189(127.25.108.190:40189)] connecting to peer
09:04:09.926 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189
09:04:09.926 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.108.190:40189(127.25.108.190:40189)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189
09:04:09.926 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.108.190:40189: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189
09:04:09.927 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.108.190:40189,127.25.108.188:38177,127.25.108.189:37911; will retry
09:04:09.985 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.25.108.190:40189(127.25.108.190:40189)] connecting to peer
09:04:09.986 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189
09:04:09.986 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.108.190:40189(127.25.108.190:40189)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189
09:04:09.986 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.108.190:40189: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189
09:04:09.987 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.108.190:40189,127.25.108.188:38177,127.25.108.189:37911; will retry
09:04:10.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:10.008105 26849 raft_consensus.cc:493] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
09:04:10.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:10.008337 26849 raft_consensus.cc:515] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aaeb9a61bdca4ab8bdc0ee318551ab52" member_type: VOTER last_known_addr { host: "127.25.108.190" port: 40189 } } peers { permanent_uuid: "0ec5572f7e8c4d27bb850d5983c8eb07" member_type: VOTER last_known_addr { host: "127.25.108.189" port: 37911 } } peers { permanent_uuid: "db0c22fe226d4071b623e4f6921f4a3d" member_type: VOTER last_known_addr { host: "127.25.108.188" port: 38177 } }
09:04:10.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:10.008942 26849 leader_election.cc:290] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers aaeb9a61bdca4ab8bdc0ee318551ab52 (127.25.108.190:40189), db0c22fe226d4071b623e4f6921f4a3d (127.25.108.188:38177)
09:04:10.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:10.009564 26247 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0ec5572f7e8c4d27bb850d5983c8eb07" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "db0c22fe226d4071b623e4f6921f4a3d" is_pre_election: true
09:04:10.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:10.009771 26247 raft_consensus.cc:2468] T 00000000000000000000000000000000 P db0c22fe226d4071b623e4f6921f4a3d [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0ec5572f7e8c4d27bb850d5983c8eb07 in term 2.
09:04:10.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:10.010223 26143 leader_election.cc:304] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0ec5572f7e8c4d27bb850d5983c8eb07, db0c22fe226d4071b623e4f6921f4a3d; no voters:
09:04:10.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:10.010399 26144 leader_election.cc:336] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer aaeb9a61bdca4ab8bdc0ee318551ab52 (127.25.108.190:40189): Network error: Client connection negotiation failed: client connection to 127.25.108.190:40189: connect: Connection refused (error 111)
09:04:10.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:10.010537 26849 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07 [term 2 FOLLOWER]: Leader pre-election won for term 3
09:04:10.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:10.010632 26849 raft_consensus.cc:493] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
09:04:10.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:10.010739 26849 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07 [term 2 FOLLOWER]: Advancing to term 3
09:04:10.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:10.012681 26849 raft_consensus.cc:515] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aaeb9a61bdca4ab8bdc0ee318551ab52" member_type: VOTER last_known_addr { host: "127.25.108.190" port: 40189 } } peers { permanent_uuid: "0ec5572f7e8c4d27bb850d5983c8eb07" member_type: VOTER last_known_addr { host: "127.25.108.189" port: 37911 } } peers { permanent_uuid: "db0c22fe226d4071b623e4f6921f4a3d" member_type: VOTER last_known_addr { host: "127.25.108.188" port: 38177 } }
09:04:10.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:10.013264 26849 leader_election.cc:290] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07 [CANDIDATE]: Term 3 election: Requested vote from peers aaeb9a61bdca4ab8bdc0ee318551ab52 (127.25.108.190:40189), db0c22fe226d4071b623e4f6921f4a3d (127.25.108.188:38177)
09:04:10.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:10.013798 26247 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0ec5572f7e8c4d27bb850d5983c8eb07" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "db0c22fe226d4071b623e4f6921f4a3d"
09:04:10.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:10.014051 26247 raft_consensus.cc:3060] T 00000000000000000000000000000000 P db0c22fe226d4071b623e4f6921f4a3d [term 2 FOLLOWER]: Advancing to term 3
09:04:10.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:10.014371 26144 leader_election.cc:336] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer aaeb9a61bdca4ab8bdc0ee318551ab52 (127.25.108.190:40189): Network error: Client connection negotiation failed: client connection to 127.25.108.190:40189: connect: Connection refused (error 111)
09:04:10.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:10.016273 26247 raft_consensus.cc:2468] T 00000000000000000000000000000000 P db0c22fe226d4071b623e4f6921f4a3d [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0ec5572f7e8c4d27bb850d5983c8eb07 in term 3.
09:04:10.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:10.016682 26143 leader_election.cc:304] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 0ec5572f7e8c4d27bb850d5983c8eb07, db0c22fe226d4071b623e4f6921f4a3d; no voters: aaeb9a61bdca4ab8bdc0ee318551ab52
09:04:10.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:10.016964 26849 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07 [term 3 FOLLOWER]: Leader election won for term 3
09:04:10.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:10.017355 26849 raft_consensus.cc:697] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07 [term 3 LEADER]: Becoming Leader. State: Replica: 0ec5572f7e8c4d27bb850d5983c8eb07, State: Running, Role: LEADER
09:04:10.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:10.017753 26849 consensus_queue.cc:237] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aaeb9a61bdca4ab8bdc0ee318551ab52" member_type: VOTER last_known_addr { host: "127.25.108.190" port: 40189 } } peers { permanent_uuid: "0ec5572f7e8c4d27bb850d5983c8eb07" member_type: VOTER last_known_addr { host: "127.25.108.189" port: 37911 } } peers { permanent_uuid: "db0c22fe226d4071b623e4f6921f4a3d" member_type: VOTER last_known_addr { host: "127.25.108.188" port: 38177 } }
09:04:10.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:10.018985 26855 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 0ec5572f7e8c4d27bb850d5983c8eb07. Latest consensus state: current_term: 3 leader_uuid: "0ec5572f7e8c4d27bb850d5983c8eb07" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aaeb9a61bdca4ab8bdc0ee318551ab52" member_type: VOTER last_known_addr { host: "127.25.108.190" port: 40189 } } peers { permanent_uuid: "0ec5572f7e8c4d27bb850d5983c8eb07" member_type: VOTER last_known_addr { host: "127.25.108.189" port: 37911 } } peers { permanent_uuid: "db0c22fe226d4071b623e4f6921f4a3d" member_type: VOTER last_known_addr { host: "127.25.108.188" port: 38177 } } }
09:04:10.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:10.019238 26855 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07 [sys.catalog]: This master's current role is: LEADER
09:04:10.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:10.019868 26856 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
09:04:10.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:10.022611 26856 catalog_manager.cc:679] Loaded metadata for table txn_manager_fallback [id=1461c31339b24f7783bf16fa929cf842]
09:04:10.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:10.023052 26856 catalog_manager.cc:679] Loaded metadata for table kudu_system.kudu_transactions [id=5ec6775cc92b49608f2878cc20b58f73]
09:04:10.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:10.024703 26856 tablet_loader.cc:96] loaded metadata for tablet 2de22c9ead36457085cf570fcc66f08b (table kudu_system.kudu_transactions [id=5ec6775cc92b49608f2878cc20b58f73])
09:04:10.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:10.025118 26856 tablet_loader.cc:96] loaded metadata for tablet 474f7edac15d4c468a54a651ead3a2a5 (table txn_manager_fallback [id=1461c31339b24f7783bf16fa929cf842])
09:04:10.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:10.025418 26856 tablet_loader.cc:96] loaded metadata for tablet 8bf23027b1c64e248061f4f4a34dc7a6 (table txn_manager_fallback [id=1461c31339b24f7783bf16fa929cf842])
09:04:10.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:10.025638 26856 catalog_manager.cc:1494] Initializing Kudu cluster ID...
09:04:10.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:10.026449 26856 catalog_manager.cc:1269] Loaded cluster ID: 8e6b4924608e40b9b93ef0e275611781
09:04:10.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:10.026558 26856 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
09:04:10.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:10.027674 26856 catalog_manager.cc:1514] Loading token signing keys...
09:04:10.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:10.028517 26856 catalog_manager.cc:6033] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07: Loaded TSK: 0
09:04:10.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:10.029321 26856 catalog_manager.cc:1524] Initializing in-progress tserver states...
09:04:10.065 [DEBUG - pool-38-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.190:40189, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.188:38177, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.189:37911, [1ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.190:40189: Network error: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189, [2ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.189:37911: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.188:38177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.25.108.190:40189,127.25.108.188:38177,127.25.108.189:37911) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.190:40189, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.188:38177, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.189:37911, [21ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.190:40189: Network error: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189, [22ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.189:37911: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.188:38177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.25.108.190:40189,127.25.108.188:38177,127.25.108.189:37911) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.190:40189, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.188:38177, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.189:37911, [42ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.190:40189: Network error: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189, [43ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.188:38177: OK, [43ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.189:37911: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.25.108.190:40189,127.25.108.188:38177,127.25.108.189:37911) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.190:40189, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.188:38177, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.189:37911, [61ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.190:40189: Network error: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189, [61ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.188:38177: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.189:37911: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.25.108.190:40189,127.25.108.188:38177,127.25.108.189:37911) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.190:40189, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.188:38177, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.189:37911, [81ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.189:37911: OK, [81ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.190:40189: Network error: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189, [82ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.188:38177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.25.108.190:40189,127.25.108.188:38177,127.25.108.189:37911) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.190:40189, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.188:38177, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.189:37911, [121ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.190:40189: Network error: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189, [121ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.188:38177: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.189:37911: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.25.108.190:40189,127.25.108.188:38177,127.25.108.189:37911) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.190:40189, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.188:38177, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.189:37911, [181ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.190:40189: Network error: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189, [182ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.188:38177: OK, [182ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.189:37911: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.25.108.190:40189,127.25.108.188:38177,127.25.108.189:37911) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189, deferred=Deferred@2120939207(state=PENDING, result=null, callback=(continuation of Deferred@2041301548 after retry RPC after error@1900474233) -> (continuation of Deferred@1283210319 after retry RPC after error@1703491529) -> (continuation of Deferred@1112833022 after retry RPC after error@70220266) -> (continuation of Deferred@698135396 after retry RPC after error@84405985) -> (continuation of Deferred@1693314612 after retry RPC after error@468855658) -> (continuation of Deferred@1471786211 after retry RPC after error@1844650003) -> (continuation of Deferred@1717891031 after retry RPC after error@1824989450), errback=(continuation of Deferred@2041301548 after retry RPC after error@1900474233) -> (continuation of Deferred@1283210319 after retry RPC after error@1703491529) -> (continuation of Deferred@1112833022 after retry RPC after error@70220266) -> (continuation of Deferred@698135396 after retry RPC after error@84405985) -> (continuation of Deferred@1693314612 after retry RPC after error@468855658) -> (continuation of Deferred@1471786211 after retry RPC after error@1844650003) -> (continuation of Deferred@1717891031 after retry RPC after error@1824989450)))
09:04:10.067 [WARN - pool-38-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
09:04:10.085 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.25.108.190:40189(127.25.108.190:40189)] connecting to peer
09:04:10.087 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189
09:04:10.087 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.108.190:40189(127.25.108.190:40189)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189
09:04:10.087 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.108.190:40189: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189
09:04:10.089 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
09:04:10.089 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=1, TimeoutTracker(timeout=250, elapsed=3), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.190:40189, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.188:38177, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.108.189:37911, [2ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.190:40189: Network error: Failed to connect to peer master-127.25.108.190:40189(127.25.108.190:40189): Connection refused: /127.25.108.190:40189, [2ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.188:38177: OK, [3ms] Sub RPC ConnectToMaster: received response from server master-127.25.108.189:37911: OK, deferred=null) after lookup
09:04:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:10.114452 26366 txn_status_manager.cc:1391] automatically aborted stale txn (ID 0) past 1.004s from last keepalive heartbeat (effective timeout is 1.000s)
09:04:10.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:10.121469 26144 consensus_peers.cc:597] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07 -> Peer aaeb9a61bdca4ab8bdc0ee318551ab52 (127.25.108.190:40189): Couldn't send request to peer aaeb9a61bdca4ab8bdc0ee318551ab52. Status: Network error: Client connection negotiation failed: client connection to 127.25.108.190:40189: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
09:04:10.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:10.126066 26403 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.25.108.190:40189)
09:04:10.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:10.149989 26247 raft_consensus.cc:1275] T 00000000000000000000000000000000 P db0c22fe226d4071b623e4f6921f4a3d [term 3 FOLLOWER]: Refusing update from remote peer 0ec5572f7e8c4d27bb850d5983c8eb07: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 3 index: 12. (index mismatch)
09:04:10.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:10.151216 26855 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07 [LEADER]: Connected to new peer: Peer: permanent_uuid: "db0c22fe226d4071b623e4f6921f4a3d" member_type: VOTER last_known_addr { host: "127.25.108.188" port: 38177 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
09:04:10.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:10.158339 26860 tablet_replica.cc:1306] Aborted: operation has been aborted: cancelling pending write operations
09:04:10.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:10.161752 26877 sys_catalog.cc:455] T 00000000000000000000000000000000 P db0c22fe226d4071b623e4f6921f4a3d [sys.catalog]: SysCatalogTable state changed. Reason: New leader 0ec5572f7e8c4d27bb850d5983c8eb07. Latest consensus state: current_term: 3 leader_uuid: "0ec5572f7e8c4d27bb850d5983c8eb07" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aaeb9a61bdca4ab8bdc0ee318551ab52" member_type: VOTER last_known_addr { host: "127.25.108.190" port: 40189 } } peers { permanent_uuid: "0ec5572f7e8c4d27bb850d5983c8eb07" member_type: VOTER last_known_addr { host: "127.25.108.189" port: 37911 } } peers { permanent_uuid: "db0c22fe226d4071b623e4f6921f4a3d" member_type: VOTER last_known_addr { host: "127.25.108.188" port: 38177 } } }
09:04:10.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:10.162060 26877 sys_catalog.cc:458] T 00000000000000000000000000000000 P db0c22fe226d4071b623e4f6921f4a3d [sys.catalog]: This master's current role is: FOLLOWER
09:04:10.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:10.176079 26855 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "0ec5572f7e8c4d27bb850d5983c8eb07" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aaeb9a61bdca4ab8bdc0ee318551ab52" member_type: VOTER last_known_addr { host: "127.25.108.190" port: 40189 } } peers { permanent_uuid: "0ec5572f7e8c4d27bb850d5983c8eb07" member_type: VOTER last_known_addr { host: "127.25.108.189" port: 37911 } } peers { permanent_uuid: "db0c22fe226d4071b623e4f6921f4a3d" member_type: VOTER last_known_addr { host: "127.25.108.188" port: 38177 } } }
09:04:10.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:10.176519 26855 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07 [sys.catalog]: This master's current role is: LEADER
09:04:10.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:10.180168 26877 sys_catalog.cc:455] T 00000000000000000000000000000000 P db0c22fe226d4071b623e4f6921f4a3d [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "0ec5572f7e8c4d27bb850d5983c8eb07" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aaeb9a61bdca4ab8bdc0ee318551ab52" member_type: VOTER last_known_addr { host: "127.25.108.190" port: 40189 } } peers { permanent_uuid: "0ec5572f7e8c4d27bb850d5983c8eb07" member_type: VOTER last_known_addr { host: "127.25.108.189" port: 37911 } } peers { permanent_uuid: "db0c22fe226d4071b623e4f6921f4a3d" member_type: VOTER last_known_addr { host: "127.25.108.188" port: 38177 } } }
09:04:10.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:10.180424 26877 sys_catalog.cc:458] T 00000000000000000000000000000000 P db0c22fe226d4071b623e4f6921f4a3d [sys.catalog]: This master's current role is: FOLLOWER
09:04:10.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:10.172662 26157 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:39030:
09:04:10.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
09:04:10.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
09:04:10.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:04:10.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
09:04:10.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
09:04:10.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:04:10.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:04:10.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:04:10.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:04:10.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:04:10.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:04:10.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:10.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:04:10.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
09:04:10.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
09:04:10.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:04:10.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:04:10.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:04:10.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:04:10.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:04:10.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:04:10.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:10.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:04:10.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
09:04:10.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
09:04:10.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:04:10.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:04:10.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:04:10.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:04:10.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:04:10.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:04:10.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:10.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:04:10.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
09:04:10.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
09:04:10.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
09:04:10.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:04:10.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:04:10.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:04:10.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:04:10.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:04:10.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:10.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:10.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
09:04:10.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
09:04:10.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
09:04:10.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
09:04:10.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:10.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
09:04:10.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
09:04:10.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:04:10.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
09:04:10.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:10.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:10.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:10.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
09:04:10.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:10.185272 26157 catalog_manager.cc:7011] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
09:04:10.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:10.187049 26769 heartbeater.cc:646] Failed to heartbeat to 127.25.108.190:40189 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.25.108.190:40189: connect: Connection refused (error 111)
09:04:10.207 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) terminating keepalive task (txn ID 0) due to exception org.apache.kudu.client.NonRecoverableException: Failed to write to server: 06ecf9cc74fc4652a29bcab5f0d1d702 (127.25.108.129:37639): transaction ID 0 is not available for further commits: state: ABORTED user: "slave" start_timestamp: 1761728649 last_transition_timestamp: 1761728650
09:04:10.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:10.701745 26144 consensus_peers.cc:597] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07 -> Peer aaeb9a61bdca4ab8bdc0ee318551ab52 (127.25.108.190:40189): Couldn't send request to peer aaeb9a61bdca4ab8bdc0ee318551ab52. Status: Network error: Client connection negotiation failed: client connection to 127.25.108.190:40189: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
09:04:11.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:11.119241 26144 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.25.108.190:40189: connect: Connection refused (error 111) [suppressed 18 similar messages]
09:04:11.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:11.120299 26144 consensus_peers.cc:597] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07 -> Peer aaeb9a61bdca4ab8bdc0ee318551ab52 (127.25.108.190:40189): Couldn't send request to peer aaeb9a61bdca4ab8bdc0ee318551ab52. Status: Network error: Client connection negotiation failed: client connection to 127.25.108.190:40189: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
09:04:11.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:11.186758 26770 heartbeater.cc:499] Master 127.25.108.189:37911 was elected leader, sending a full tablet report...
09:04:11.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:11.196161 26629 heartbeater.cc:499] Master 127.25.108.189:37911 was elected leader, sending a full tablet report...
09:04:11.367 [DEBUG - main] (Connection.java:575) [peer master-127.25.108.190:40189(127.25.108.190:40189)] disconnecting while in state TERMINATED
09:04:11.367 [DEBUG - main] (Connection.java:575) [peer master-127.25.108.188:38177(127.25.108.188:38177)] disconnecting while in state READY
09:04:11.367 [DEBUG - main] (Connection.java:575) [peer a2ba097a80234997bff89ca113391c46(127.25.108.130:43701)] disconnecting while in state READY
09:04:11.367 [DEBUG - main] (Connection.java:575) [peer master-127.25.108.189:37911(127.25.108.189:37911)] disconnecting while in state READY
09:04:11.368 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.25.108.188:38177(127.25.108.188:38177)] handling channelInactive
09:04:11.368 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
09:04:11.368 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.108.188:38177(127.25.108.188:38177)] cleaning up while in state READY due to: connection closed
09:04:11.369 [DEBUG - kudu-nio-1] (Connection.java:239) [peer a2ba097a80234997bff89ca113391c46(127.25.108.130:43701)] handling channelInactive
09:04:11.369 [DEBUG - kudu-nio-1] (Connection.java:709) [peer a2ba097a80234997bff89ca113391c46(127.25.108.130:43701)] cleaning up while in state READY due to: connection closed
09:04:11.369 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.25.108.189:37911(127.25.108.189:37911)] handling channelInactive
09:04:11.369 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.108.189:37911(127.25.108.189:37911)] cleaning up while in state READY due to: connection closed
09:04:11.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:11.369350 26034 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskescuMK/build/asan/bin/kudu pid 26327
09:04:11.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:11.381196 26479 generic_service.cc:196] Checking for leaks (request via RPC)
09:04:11.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:11.509395 26034 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskescuMK/build/asan/bin/kudu with pid 26327
09:04:11.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:11.531308 26034 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskescuMK/build/asan/bin/kudu pid 26489
09:04:11.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:11.538128 26622 generic_service.cc:196] Checking for leaks (request via RPC)
09:04:11.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:11.593750 26204 master_runner.cc:186] Error getting master registration for 127.25.108.190:40189: Network error: Client connection negotiation failed: client connection to 127.25.108.190:40189: connect: Connection refused (error 111),
09:04:11.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:11.594013 26204 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
09:04:11.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:11.615952 26144 consensus_peers.cc:597] T 00000000000000000000000000000000 P 0ec5572f7e8c4d27bb850d5983c8eb07 -> Peer aaeb9a61bdca4ab8bdc0ee318551ab52 (127.25.108.190:40189): Couldn't send request to peer aaeb9a61bdca4ab8bdc0ee318551ab52. Status: Network error: Client connection negotiation failed: client connection to 127.25.108.190:40189: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
09:04:11.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:11.637594 26034 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskescuMK/build/asan/bin/kudu with pid 26489
09:04:11.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:11.654932 26648 connection.cc:537] server connection from 127.25.108.130:35531 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
09:04:11.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:11.655303 26034 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskescuMK/build/asan/bin/kudu pid 26632
09:04:11.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:11.662300 26763 generic_service.cc:196] Checking for leaks (request via RPC)
09:04:11.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:11.758464 26034 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskescuMK/build/asan/bin/kudu with pid 26632
09:04:11.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:11.776238 26034 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskescuMK/build/asan/bin/kudu pid 26125
09:04:11.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:11.777405 26197 generic_service.cc:196] Checking for leaks (request via RPC)
09:04:11.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:11.890487 26034 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskescuMK/build/asan/bin/kudu with pid 26125
09:04:11.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:04:11.910950 26224 connection.cc:537] server connection from 127.0.0.1:42724 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
09:04:11.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:11.911166 26034 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskescuMK/build/asan/bin/kudu pid 26204
09:04:11.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:11.912238 26278 generic_service.cc:196] Checking for leaks (request via RPC)
09:04:12.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:04:12.019677 26034 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskescuMK/build/asan/bin/kudu with pid 26204
09:04:12.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-29T09:04:12Z chronyd exiting
09:04:12.173 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveSwitchesToOtherTxnManager: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Failed to write to server: 06ecf9cc74fc4652a29bcab5f0d1d702 (127.25.108.129:37639): transaction ID 0 is not open: state: ABORTED user: "slave" start_timestamp: 1761728649 last_transition_timestamp: 1761728650
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduTransaction.doCommitTransaction(KuduTransaction.java:491) ~[main/:?]
at org.apache.kudu.client.KuduTransaction.commitWithMode(KuduTransaction.java:537) ~[main/:?]
at org.apache.kudu.client.KuduTransaction.commit(KuduTransaction.java:284) ~[main/:?]
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveSwitchesToOtherTxnManager(TestKuduTransaction.java:1152) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.RpcProxy.dispatchTxnManagerError(RpcProxy.java:439) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:298) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1503) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1366) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1415) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Failed to write to server: 06ecf9cc74fc4652a29bcab5f0d1d702 (127.25.108.129:37639): transaction ID 0 is not open: state: ABORTED user: "slave" start_timestamp: 1761728649 last_transition_timestamp: 1761728650
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduTransaction.doCommitTransaction(KuduTransaction.java:491)
at org.apache.kudu.client.KuduTransaction.commitWithMode(KuduTransaction.java:537)
at org.apache.kudu.client.KuduTransaction.commit(KuduTransaction.java:284)
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveSwitchesToOtherTxnManager(TestKuduTransaction.java:1152)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.RpcProxy.dispatchTxnManagerError(RpcProxy.java:439)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:298)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1503)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1366)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1415)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more