08:03:20.723 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
08:03:20.723 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskbKfKhV/test-tmp
08:03:20.724 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/asan/bin
08:03:20.724 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskbKfKhV/java/build/dist-test/../../../build/asan/bin/kudu, test, mini_cluster, --serialization=pb]
08:03:20.725 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382"
extra_master_flags: "--txn_manager_enabled"
extra_master_flags: "--raft_heartbeat_interval_ms=100"
extra_tserver_flags: "--txn_keepalive_interval_ms=1000"
extra_tserver_flags: "--txn_staleness_tracker_interval_ms=250"
extra_tserver_flags: "--enable_txn_system_client_init=true"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
08:03:21.053 [DEBUG - main] (MiniKuduCluster.java:195) Response:
08:03:21.053 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
08:03:21.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.056047 16034 mini_oidc.cc:150] Starting JWKS server
08:03:21.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.058568 16034 webserver.cc:466] Webserver started at http://127.0.0.1:42281/ using document root <none> and password file <none>
08:03:21.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.058794 16034 mini_oidc.cc:186] Starting OIDC Discovery server
08:03:21.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.059392 16034 webserver.cc:466] Webserver started at http://127.0.0.1:39193/ using document root <none> and password file <none>
08:03:21.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-04-10T08:03:21Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
08:03:21.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-04-10T08:03:21Z Disabled control of system clock
08:03:21.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.091809 16034 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskbKfKhV/build/asan/bin/kudu
08:03:21.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbKfKhV/build/asan/bin/kudu
08:03:21.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/master-0/wal
08:03:21.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/master-0/data
08:03:21.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:21.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:21.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:21.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:21.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:21.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:21.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/master-0/logs
08:03:21.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/master-0/data/info.pb
08:03:21.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:21.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:21.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:21.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:21.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:21.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:21.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:21.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:21.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:03:21.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:21.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:03:21.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:03:21.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.168.190:33711
08:03:21.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.168.190
08:03:21.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:21.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.168.148:38479
08:03:21.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:21.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:21.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:21.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:03:21.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.168.190:33711,127.15.168.189:43117,127.15.168.188:38185
08:03:21.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:03:21.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42281/jwks.json
08:03:21.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
08:03:21.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
08:03:21.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:21.417088 16045 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:03:21.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:21.417454 16045 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:21.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:21.417542 16045 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:03:21.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:21.425855 16045 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:03:21.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:21.426193 16045 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
08:03:21.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:21.426306 16045 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:03:21.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:21.426378 16045 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:03:21.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:21.426465 16045 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:03:21.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:21.426548 16045 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:03:21.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:21.426640 16045 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42281/jwks.json
08:03:21.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.436820 16045 master_runner.cc:386] Master server non-default flags:
08:03:21.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:21.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.168.148:38479
08:03:21.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:21.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:21.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
08:03:21.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/master-0/data
08:03:21.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/master-0/wal
08:03:21.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:03:21.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.168.190:33711,127.15.168.189:43117,127.15.168.188:38185
08:03:21.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
08:03:21.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:21.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:21.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:03:21.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.168.190:33711
08:03:21.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:03:21.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:21.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:03:21.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42281/jwks.json
08:03:21.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:21.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:21.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/master-0/data/info.pb
08:03:21.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.168.190
08:03:21.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:21.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:21.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:21.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:21.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:21.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:21.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/master-0/logs
08:03:21.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:21.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:21.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:03:21.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:03:21.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:03:21.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 586cfc20905df35f49541b53af4442736790b0c1
08:03:21.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:03:21.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Apr 2025 07:43:21 UTC on 24a791456cd2
08:03:21.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 5550
08:03:21.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
08:03:21.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.438850 16045 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:21.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.441059 16045 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:21.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:21.452100 16053 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:21.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:21.452237 16051 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:21.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:21.452209 16050 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:21.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.453399 16045 server_base.cc:1034] running on GCE node
08:03:21.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.454241 16045 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:21.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.455811 16045 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:21.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.457054 16045 hybrid_clock.cc:648] HybridClock initialized: now 1744272201456982 us; error 43 us; skew 500 ppm
08:03:21.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.457427 16045 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:03:21.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.459975 16045 webserver.cc:466] Webserver started at http://127.15.168.190:44023/ using document root <none> and password file <none>
08:03:21.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.460731 16045 fs_manager.cc:362] Metadata directory not provided
08:03:21.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.460858 16045 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:21.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.461234 16045 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:03:21.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.463778 16045 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/master-0/data/instance:
08:03:21.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2a06e94e44f24a35b5da6da522938c92"
08:03:21.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-04-10 08:03:21 on dist-test-slave-cb5c"
08:03:21.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.464581 16045 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/master-0/wal/instance:
08:03:21.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2a06e94e44f24a35b5da6da522938c92"
08:03:21.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-04-10 08:03:21 on dist-test-slave-cb5c"
08:03:21.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.470238 16045 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.006s sys 0.000s
08:03:21.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.473842 16059 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:03:21.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.475296 16045 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
08:03:21.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.475476 16045 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/master-0/data,/tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/master-0/wal
08:03:21.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2a06e94e44f24a35b5da6da522938c92"
08:03:21.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-04-10 08:03:21 on dist-test-slave-cb5c"
08:03:21.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.475696 16045 fs_report.cc:389] FS layout report
08:03:21.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:21.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/master-0/wal
08:03:21.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/master-0/wal
08:03:21.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/master-0/data/data
08:03:21.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:21.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:21.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:21.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:21.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:21.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:21.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:21.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:21.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:21.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:21.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:21.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.491514 16045 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:03:21.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.492451 16045 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:21.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.492756 16045 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:21.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.516511 16045 rpc_server.cc:307] RPC server started. Bound to: 127.15.168.190:33711
08:03:21.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.516564 16120 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.15.168.190:33711 every 8 connection(s)
08:03:21.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.518054 16045 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/master-0/data/info.pb
08:03:21.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.520308 16034 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskbKfKhV/build/asan/bin/kudu as pid 16045
08:03:21.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.520606 16034 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/master-0/wal/instance
08:03:21.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.521944 16121 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:21.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.522941 16034 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskbKfKhV/build/asan/bin/kudu
08:03:21.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbKfKhV/build/asan/bin/kudu
08:03:21.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/master-1/wal
08:03:21.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/master-1/data
08:03:21.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:21.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:21.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:21.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:21.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:21.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:21.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/master-1/logs
08:03:21.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/master-1/data/info.pb
08:03:21.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:21.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:21.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:21.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:21.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:21.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:21.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:21.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:21.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:03:21.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:21.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:03:21.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:03:21.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.168.189:43117
08:03:21.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.168.189
08:03:21.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:21.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.168.148:38479
08:03:21.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:21.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:21.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:21.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:03:21.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.168.190:33711,127.15.168.189:43117,127.15.168.188:38185
08:03:21.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:03:21.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42281/jwks.json
08:03:21.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
08:03:21.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
08:03:21.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.527927 16121 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.168.190" port: 33711 } has no permanent_uuid. Determining permanent_uuid...
08:03:21.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.545650 16121 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.168.189" port: 43117 } has no permanent_uuid. Determining permanent_uuid...
08:03:21.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:21.546924 16063 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.15.168.189:43117: connect: Connection refused (error 111)
08:03:21.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:21.548604 16121 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.168.189:43117: Network error: Client connection negotiation failed: client connection to 127.15.168.189:43117: connect: Connection refused (error 111)
08:03:21.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.597977 16121 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.168.189" port: 43117 } attempt: 1
08:03:21.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:21.600109 16121 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.168.189:43117: Network error: Client connection negotiation failed: client connection to 127.15.168.189:43117: connect: Connection refused (error 111)
08:03:21.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.668403 16121 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.168.189" port: 43117 } attempt: 2
08:03:21.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:21.670619 16121 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.168.189:43117: Network error: Client connection negotiation failed: client connection to 127.15.168.189:43117: connect: Connection refused (error 111)
08:03:21.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.761897 16121 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.168.189" port: 43117 } attempt: 3
08:03:21.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:21.763906 16121 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.168.189:43117: Network error: Client connection negotiation failed: client connection to 127.15.168.189:43117: connect: Connection refused (error 111)
08:03:21.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:21.858204 16123 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:03:21.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:21.858587 16123 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:21.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:21.858669 16123 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:03:21.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:21.868415 16123 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:03:21.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:21.868557 16123 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
08:03:21.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:21.868635 16123 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:03:21.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:21.868696 16123 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:03:21.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:21.868777 16123 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:03:21.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:21.868827 16123 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:03:21.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:21.868896 16123 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42281/jwks.json
08:03:21.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.879477 16123 master_runner.cc:386] Master server non-default flags:
08:03:21.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:21.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.168.148:38479
08:03:21.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:21.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:21.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
08:03:21.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/master-1/data
08:03:21.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/master-1/wal
08:03:21.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:03:21.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.168.190:33711,127.15.168.189:43117,127.15.168.188:38185
08:03:21.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
08:03:21.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:21.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:21.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:03:21.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.168.189:43117
08:03:21.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:03:21.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:21.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:03:21.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42281/jwks.json
08:03:21.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:21.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:21.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/master-1/data/info.pb
08:03:21.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.168.189
08:03:21.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:21.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:21.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:21.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:21.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:21.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:21.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/master-1/logs
08:03:21.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:21.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:21.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:03:21.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:03:21.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:03:21.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 586cfc20905df35f49541b53af4442736790b0c1
08:03:21.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:03:21.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Apr 2025 07:43:21 UTC on 24a791456cd2
08:03:21.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 5550
08:03:21.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
08:03:21.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.881201 16123 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:21.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.883080 16123 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:21.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:21.894083 16132 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:21.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:21.894173 16131 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:21.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:21.894127 16134 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:21.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.895227 16123 server_base.cc:1034] running on GCE node
08:03:21.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.896035 16123 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:21.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.897531 16123 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:21.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.898761 16123 hybrid_clock.cc:648] HybridClock initialized: now 1744272201898700 us; error 45 us; skew 500 ppm
08:03:21.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.899083 16123 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:03:21.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.901558 16123 webserver.cc:466] Webserver started at http://127.15.168.189:43431/ using document root <none> and password file <none>
08:03:21.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.902308 16123 fs_manager.cc:362] Metadata directory not provided
08:03:21.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.902419 16123 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:21.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.902719 16123 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:03:21.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.905294 16123 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/master-1/data/instance:
08:03:21.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e84d2a2c5f154a8f8da59be47f9d54db"
08:03:21.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-04-10 08:03:21 on dist-test-slave-cb5c"
08:03:21.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.906042 16123 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/master-1/wal/instance:
08:03:21.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e84d2a2c5f154a8f8da59be47f9d54db"
08:03:21.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-04-10 08:03:21 on dist-test-slave-cb5c"
08:03:21.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.907182 16121 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.168.189" port: 43117 } attempt: 4
08:03:21.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:21.909116 16121 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.168.189:43117: Network error: Client connection negotiation failed: client connection to 127.15.168.189:43117: connect: Connection refused (error 111)
08:03:21.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.912328 16123 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.007s sys 0.001s
08:03:21.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.916123 16140 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:03:21.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.917752 16123 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.001s
08:03:21.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.917927 16123 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/master-1/data,/tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/master-1/wal
08:03:21.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e84d2a2c5f154a8f8da59be47f9d54db"
08:03:21.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-04-10 08:03:21 on dist-test-slave-cb5c"
08:03:21.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.918116 16123 fs_report.cc:389] FS layout report
08:03:21.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:21.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/master-1/wal
08:03:21.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/master-1/wal
08:03:21.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/master-1/data/data
08:03:21.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:21.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:21.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:21.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:21.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:21.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:21.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:21.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:21.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:21.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:21.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:21.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.935603 16123 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:03:21.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.936480 16123 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:21.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.936728 16123 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:21.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.960408 16123 rpc_server.cc:307] RPC server started. Bound to: 127.15.168.189:43117
08:03:21.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.960448 16201 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.15.168.189:43117 every 8 connection(s)
08:03:21.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.961918 16123 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/master-1/data/info.pb
08:03:21.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.965860 16202 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:21.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.972025 16034 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskbKfKhV/build/asan/bin/kudu as pid 16123
08:03:21.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.972208 16034 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/master-1/wal/instance
08:03:21.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.972271 16202 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.168.190" port: 33711 } has no permanent_uuid. Determining permanent_uuid...
08:03:21.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.973506 16034 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskbKfKhV/build/asan/bin/kudu
08:03:21.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbKfKhV/build/asan/bin/kudu
08:03:21.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/master-2/wal
08:03:21.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/master-2/data
08:03:21.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:21.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:21.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:21.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:21.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:21.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:21.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/master-2/logs
08:03:21.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/master-2/data/info.pb
08:03:21.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:21.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:21.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:21.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:21.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:21.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:21.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:21.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:21.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:03:21.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:21.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:03:21.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:03:21.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.168.188:38185
08:03:21.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.168.188
08:03:21.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:21.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.168.148:38479
08:03:21.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:21.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:21.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:21.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:03:21.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.168.190:33711,127.15.168.189:43117,127.15.168.188:38185
08:03:21.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:03:21.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42281/jwks.json
08:03:21.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
08:03:21.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
08:03:21.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.987231 16202 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.168.189" port: 43117 } has no permanent_uuid. Determining permanent_uuid...
08:03:21.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:21.997900 16202 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.168.188" port: 38185 } has no permanent_uuid. Determining permanent_uuid...
08:03:21.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:21.999064 16141 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.15.168.188:38185: connect: Connection refused (error 111)
08:03:22.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:22.000882 16202 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.168.188:38185: Network error: Client connection negotiation failed: client connection to 127.15.168.188:38185: connect: Connection refused (error 111)
08:03:22.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.050316 16202 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.168.188" port: 38185 } attempt: 1
08:03:22.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:22.052278 16202 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.168.188:38185: Network error: Client connection negotiation failed: client connection to 127.15.168.188:38185: connect: Connection refused (error 111)
08:03:22.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.120509 16202 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.168.188" port: 38185 } attempt: 2
08:03:22.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:22.122567 16202 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.168.188:38185: Network error: Client connection negotiation failed: client connection to 127.15.168.188:38185: connect: Connection refused (error 111)
08:03:22.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.208374 16121 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.168.189" port: 43117 } attempt: 5
08:03:22.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.213795 16202 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.168.188" port: 38185 } attempt: 3
08:03:22.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.215123 16121 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.168.188" port: 38185 } has no permanent_uuid. Determining permanent_uuid...
08:03:22.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:22.216300 16202 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.168.188:38185: Network error: Client connection negotiation failed: client connection to 127.15.168.188:38185: connect: Connection refused (error 111)
08:03:22.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:22.217054 16121 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.168.188:38185: Network error: Client connection negotiation failed: client connection to 127.15.168.188:38185: connect: Connection refused (error 111)
08:03:22.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.268277 16121 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.168.188" port: 38185 } attempt: 1
08:03:22.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:22.270267 16121 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.168.188:38185: Network error: Client connection negotiation failed: client connection to 127.15.168.188:38185: connect: Connection refused (error 111)
08:03:22.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:22.303251 16205 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:03:22.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:22.303614 16205 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:22.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:22.303701 16205 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:03:22.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:22.311817 16205 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:03:22.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:22.311928 16205 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
08:03:22.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:22.311978 16205 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:03:22.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:22.312021 16205 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:03:22.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:22.312075 16205 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:03:22.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:22.312115 16205 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:03:22.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:22.312158 16205 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42281/jwks.json
08:03:22.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.322019 16205 master_runner.cc:386] Master server non-default flags:
08:03:22.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:22.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.168.148:38479
08:03:22.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:22.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:22.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
08:03:22.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/master-2/data
08:03:22.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/master-2/wal
08:03:22.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:03:22.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.168.190:33711,127.15.168.189:43117,127.15.168.188:38185
08:03:22.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
08:03:22.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:22.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:22.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:03:22.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.168.188:38185
08:03:22.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:03:22.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:22.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:03:22.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42281/jwks.json
08:03:22.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:22.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:22.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/master-2/data/info.pb
08:03:22.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.168.188
08:03:22.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:22.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:22.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:22.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:22.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:22.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:22.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/master-2/logs
08:03:22.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:22.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:22.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:03:22.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:03:22.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:03:22.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 586cfc20905df35f49541b53af4442736790b0c1
08:03:22.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:03:22.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Apr 2025 07:43:21 UTC on 24a791456cd2
08:03:22.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 5550
08:03:22.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
08:03:22.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.323702 16205 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:22.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.325528 16205 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:22.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:22.336700 16212 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:22.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:22.336766 16215 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:22.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:22.336763 16213 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:22.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.337772 16205 server_base.cc:1034] running on GCE node
08:03:22.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.338470 16121 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.168.188" port: 38185 } attempt: 2
08:03:22.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.338611 16205 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:22.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:22.340163 16121 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.168.188:38185: Network error: Client connection negotiation failed: client connection to 127.15.168.188:38185: connect: Connection refused (error 111)
08:03:22.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.340247 16205 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:22.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.341468 16205 hybrid_clock.cc:648] HybridClock initialized: now 1744272202341391 us; error 51 us; skew 500 ppm
08:03:22.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.341785 16205 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:03:22.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.344293 16205 webserver.cc:466] Webserver started at http://127.15.168.188:36971/ using document root <none> and password file <none>
08:03:22.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.345039 16205 fs_manager.cc:362] Metadata directory not provided
08:03:22.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.345149 16205 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:22.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.345441 16205 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:03:22.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.348044 16205 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/master-2/data/instance:
08:03:22.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5bee3c957fe9436399c523d7516cc493"
08:03:22.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-04-10 08:03:22 on dist-test-slave-cb5c"
08:03:22.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.348819 16205 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/master-2/wal/instance:
08:03:22.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5bee3c957fe9436399c523d7516cc493"
08:03:22.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-04-10 08:03:22 on dist-test-slave-cb5c"
08:03:22.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.354323 16205 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.005s sys 0.001s
08:03:22.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.357807 16221 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:03:22.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.359303 16205 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.001s sys 0.002s
08:03:22.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.359462 16205 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/master-2/data,/tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/master-2/wal
08:03:22.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5bee3c957fe9436399c523d7516cc493"
08:03:22.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-04-10 08:03:22 on dist-test-slave-cb5c"
08:03:22.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.359510 16202 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.168.188" port: 38185 } attempt: 4
08:03:22.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.359647 16205 fs_report.cc:389] FS layout report
08:03:22.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:22.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/master-2/wal
08:03:22.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/master-2/wal
08:03:22.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/master-2/data/data
08:03:22.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:22.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:22.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:22.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:22.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:22.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:22.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:22.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:22.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:22.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:22.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:22.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:22.361320 16202 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.168.188:38185: Network error: Client connection negotiation failed: client connection to 127.15.168.188:38185: connect: Connection refused (error 111)
08:03:22.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.377456 16205 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:03:22.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.378338 16205 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:22.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.378582 16205 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:22.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.403023 16205 rpc_server.cc:307] RPC server started. Bound to: 127.15.168.188:38185
08:03:22.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.403072 16282 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.15.168.188:38185 every 8 connection(s)
08:03:22.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.404621 16205 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/master-2/data/info.pb
08:03:22.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.408547 16283 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:22.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.412539 16034 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskbKfKhV/build/asan/bin/kudu as pid 16205
08:03:22.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.412706 16034 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/master-2/wal/instance
08:03:22.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.414719 16283 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.168.190" port: 33711 } has no permanent_uuid. Determining permanent_uuid...
08:03:22.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.432714 16283 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.168.189" port: 43117 } has no permanent_uuid. Determining permanent_uuid...
08:03:22.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.440465 16283 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.168.188" port: 38185 } has no permanent_uuid. Determining permanent_uuid...
08:03:22.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.446410 16121 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.168.188" port: 38185 } attempt: 3
08:03:22.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.452708 16283 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493: Bootstrap starting.
08:03:22.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.456238 16283 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493: Neither blocks nor log segments found. Creating new log.
08:03:22.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.457187 16121 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2a06e94e44f24a35b5da6da522938c92: Bootstrap starting.
08:03:22.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.457327 16283 log.cc:826] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493: Log is configured to *not* fsync() on all Append() calls
08:03:22.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.460317 16283 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493: No bootstrap required, opened a new log
08:03:22.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.460908 16121 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 2a06e94e44f24a35b5da6da522938c92: Neither blocks nor log segments found. Creating new log.
08:03:22.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.461972 16121 log.cc:826] T 00000000000000000000000000000000 P 2a06e94e44f24a35b5da6da522938c92: Log is configured to *not* fsync() on all Append() calls
08:03:22.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.464638 16121 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2a06e94e44f24a35b5da6da522938c92: No bootstrap required, opened a new log
08:03:22.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.466130 16283 raft_consensus.cc:357] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2a06e94e44f24a35b5da6da522938c92" member_type: VOTER last_known_addr { host: "127.15.168.190" port: 33711 } } peers { permanent_uuid: "e84d2a2c5f154a8f8da59be47f9d54db" member_type: VOTER last_known_addr { host: "127.15.168.189" port: 43117 } } peers { permanent_uuid: "5bee3c957fe9436399c523d7516cc493" member_type: VOTER last_known_addr { host: "127.15.168.188" port: 38185 } }
08:03:22.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.466435 16283 raft_consensus.cc:383] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:22.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.466570 16283 raft_consensus.cc:738] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5bee3c957fe9436399c523d7516cc493, State: Initialized, Role: FOLLOWER
08:03:22.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.467291 16283 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2a06e94e44f24a35b5da6da522938c92" member_type: VOTER last_known_addr { host: "127.15.168.190" port: 33711 } } peers { permanent_uuid: "e84d2a2c5f154a8f8da59be47f9d54db" member_type: VOTER last_known_addr { host: "127.15.168.189" port: 43117 } } peers { permanent_uuid: "5bee3c957fe9436399c523d7516cc493" member_type: VOTER last_known_addr { host: "127.15.168.188" port: 38185 } }
08:03:22.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.468250 16293 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2a06e94e44f24a35b5da6da522938c92" member_type: VOTER last_known_addr { host: "127.15.168.190" port: 33711 } } peers { permanent_uuid: "e84d2a2c5f154a8f8da59be47f9d54db" member_type: VOTER last_known_addr { host: "127.15.168.189" port: 43117 } } peers { permanent_uuid: "5bee3c957fe9436399c523d7516cc493" member_type: VOTER last_known_addr { host: "127.15.168.188" port: 38185 } } }
08:03:22.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.468551 16293 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493 [sys.catalog]: This master's current role is: FOLLOWER
08:03:22.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.469312 16283 sys_catalog.cc:564] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493 [sys.catalog]: configured and running, proceeding with master startup.
08:03:22.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.470172 16121 raft_consensus.cc:357] T 00000000000000000000000000000000 P 2a06e94e44f24a35b5da6da522938c92 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2a06e94e44f24a35b5da6da522938c92" member_type: VOTER last_known_addr { host: "127.15.168.190" port: 33711 } } peers { permanent_uuid: "e84d2a2c5f154a8f8da59be47f9d54db" member_type: VOTER last_known_addr { host: "127.15.168.189" port: 43117 } } peers { permanent_uuid: "5bee3c957fe9436399c523d7516cc493" member_type: VOTER last_known_addr { host: "127.15.168.188" port: 38185 } }
08:03:22.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.470616 16121 raft_consensus.cc:383] T 00000000000000000000000000000000 P 2a06e94e44f24a35b5da6da522938c92 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:22.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.470834 16121 raft_consensus.cc:738] T 00000000000000000000000000000000 P 2a06e94e44f24a35b5da6da522938c92 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2a06e94e44f24a35b5da6da522938c92, State: Initialized, Role: FOLLOWER
08:03:22.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.471812 16121 consensus_queue.cc:260] T 00000000000000000000000000000000 P 2a06e94e44f24a35b5da6da522938c92 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2a06e94e44f24a35b5da6da522938c92" member_type: VOTER last_known_addr { host: "127.15.168.190" port: 33711 } } peers { permanent_uuid: "e84d2a2c5f154a8f8da59be47f9d54db" member_type: VOTER last_known_addr { host: "127.15.168.189" port: 43117 } } peers { permanent_uuid: "5bee3c957fe9436399c523d7516cc493" member_type: VOTER last_known_addr { host: "127.15.168.188" port: 38185 } }
08:03:22.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.473253 16297 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2a06e94e44f24a35b5da6da522938c92 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2a06e94e44f24a35b5da6da522938c92" member_type: VOTER last_known_addr { host: "127.15.168.190" port: 33711 } } peers { permanent_uuid: "e84d2a2c5f154a8f8da59be47f9d54db" member_type: VOTER last_known_addr { host: "127.15.168.189" port: 43117 } } peers { permanent_uuid: "5bee3c957fe9436399c523d7516cc493" member_type: VOTER last_known_addr { host: "127.15.168.188" port: 38185 } } }
08:03:22.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.473567 16297 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2a06e94e44f24a35b5da6da522938c92 [sys.catalog]: This master's current role is: FOLLOWER
08:03:22.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.474329 16121 sys_catalog.cc:564] T 00000000000000000000000000000000 P 2a06e94e44f24a35b5da6da522938c92 [sys.catalog]: configured and running, proceeding with master startup.
08:03:22.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:22.482436 16307 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:03:22.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:22.482584 16307 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:03:22.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.478410 16205 master_runner.cc:186] Error getting master registration for 127.15.168.190:33711: OK, instance_id { permanent_uuid: "2a06e94e44f24a35b5da6da522938c92" instance_seqno: 1744272201511047 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:03:22.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.485241 16205 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:03:22.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:22.489209 16317 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 2a06e94e44f24a35b5da6da522938c92: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:03:22.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:22.489532 16317 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:03:22.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:22.495579 16065 tablet.cc:2367] T 00000000000000000000000000000000 P 2a06e94e44f24a35b5da6da522938c92: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:03:22.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.488183 16045 master_runner.cc:186] Error getting master registration for 127.15.168.189:43117: OK, instance_id { permanent_uuid: "e84d2a2c5f154a8f8da59be47f9d54db" instance_seqno: 1744272201954674 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:03:22.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.496836 16045 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:03:22.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.533053 16297 raft_consensus.cc:491] T 00000000000000000000000000000000 P 2a06e94e44f24a35b5da6da522938c92 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:03:22.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.533346 16297 raft_consensus.cc:513] T 00000000000000000000000000000000 P 2a06e94e44f24a35b5da6da522938c92 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2a06e94e44f24a35b5da6da522938c92" member_type: VOTER last_known_addr { host: "127.15.168.190" port: 33711 } } peers { permanent_uuid: "e84d2a2c5f154a8f8da59be47f9d54db" member_type: VOTER last_known_addr { host: "127.15.168.189" port: 43117 } } peers { permanent_uuid: "5bee3c957fe9436399c523d7516cc493" member_type: VOTER last_known_addr { host: "127.15.168.188" port: 38185 } }
08:03:22.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.536033 16248 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2a06e94e44f24a35b5da6da522938c92" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5bee3c957fe9436399c523d7516cc493" is_pre_election: true
08:03:22.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.536404 16297 leader_election.cc:290] T 00000000000000000000000000000000 P 2a06e94e44f24a35b5da6da522938c92 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e84d2a2c5f154a8f8da59be47f9d54db (127.15.168.189:43117), 5bee3c957fe9436399c523d7516cc493 (127.15.168.188:38185)
08:03:22.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.536631 16248 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2a06e94e44f24a35b5da6da522938c92 in term 0.
08:03:22.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.537334 16060 leader_election.cc:304] T 00000000000000000000000000000000 P 2a06e94e44f24a35b5da6da522938c92 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2a06e94e44f24a35b5da6da522938c92, 5bee3c957fe9436399c523d7516cc493; no voters:
08:03:22.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.537783 16297 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 2a06e94e44f24a35b5da6da522938c92 [term 0 FOLLOWER]: Leader pre-election won for term 1
08:03:22.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.537940 16297 raft_consensus.cc:491] T 00000000000000000000000000000000 P 2a06e94e44f24a35b5da6da522938c92 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:03:22.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.538071 16297 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 2a06e94e44f24a35b5da6da522938c92 [term 0 FOLLOWER]: Advancing to term 1
08:03:22.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.541471 16297 raft_consensus.cc:513] T 00000000000000000000000000000000 P 2a06e94e44f24a35b5da6da522938c92 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2a06e94e44f24a35b5da6da522938c92" member_type: VOTER last_known_addr { host: "127.15.168.190" port: 33711 } } peers { permanent_uuid: "e84d2a2c5f154a8f8da59be47f9d54db" member_type: VOTER last_known_addr { host: "127.15.168.189" port: 43117 } } peers { permanent_uuid: "5bee3c957fe9436399c523d7516cc493" member_type: VOTER last_known_addr { host: "127.15.168.188" port: 38185 } }
08:03:22.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.542927 16248 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2a06e94e44f24a35b5da6da522938c92" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5bee3c957fe9436399c523d7516cc493"
08:03:22.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.543182 16248 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493 [term 0 FOLLOWER]: Advancing to term 1
08:03:22.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.546742 16248 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2a06e94e44f24a35b5da6da522938c92 in term 1.
08:03:22.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.547361 16060 leader_election.cc:304] T 00000000000000000000000000000000 P 2a06e94e44f24a35b5da6da522938c92 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2a06e94e44f24a35b5da6da522938c92, 5bee3c957fe9436399c523d7516cc493; no voters:
08:03:22.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.548274 16297 leader_election.cc:290] T 00000000000000000000000000000000 P 2a06e94e44f24a35b5da6da522938c92 [CANDIDATE]: Term 1 election: Requested vote from peers e84d2a2c5f154a8f8da59be47f9d54db (127.15.168.189:43117), 5bee3c957fe9436399c523d7516cc493 (127.15.168.188:38185)
08:03:22.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.549307 16319 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 2a06e94e44f24a35b5da6da522938c92 [term 1 FOLLOWER]: Leader election won for term 1
08:03:22.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.550161 16319 raft_consensus.cc:695] T 00000000000000000000000000000000 P 2a06e94e44f24a35b5da6da522938c92 [term 1 LEADER]: Becoming Leader. State: Replica: 2a06e94e44f24a35b5da6da522938c92, State: Running, Role: LEADER
08:03:22.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.551815 16319 consensus_queue.cc:237] T 00000000000000000000000000000000 P 2a06e94e44f24a35b5da6da522938c92 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2a06e94e44f24a35b5da6da522938c92" member_type: VOTER last_known_addr { host: "127.15.168.190" port: 33711 } } peers { permanent_uuid: "e84d2a2c5f154a8f8da59be47f9d54db" member_type: VOTER last_known_addr { host: "127.15.168.189" port: 43117 } } peers { permanent_uuid: "5bee3c957fe9436399c523d7516cc493" member_type: VOTER last_known_addr { host: "127.15.168.188" port: 38185 } }
08:03:22.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.556372 16297 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2a06e94e44f24a35b5da6da522938c92 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 2a06e94e44f24a35b5da6da522938c92. Latest consensus state: current_term: 1 leader_uuid: "2a06e94e44f24a35b5da6da522938c92" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2a06e94e44f24a35b5da6da522938c92" member_type: VOTER last_known_addr { host: "127.15.168.190" port: 33711 } } peers { permanent_uuid: "e84d2a2c5f154a8f8da59be47f9d54db" member_type: VOTER last_known_addr { host: "127.15.168.189" port: 43117 } } peers { permanent_uuid: "5bee3c957fe9436399c523d7516cc493" member_type: VOTER last_known_addr { host: "127.15.168.188" port: 38185 } } }
08:03:22.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.558068 16297 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2a06e94e44f24a35b5da6da522938c92 [sys.catalog]: This master's current role is: LEADER
08:03:22.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.569535 16321 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
08:03:22.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.573442 16321 catalog_manager.cc:1486] Initializing Kudu cluster ID...
08:03:22.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.589709 16248 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493 [term 1 FOLLOWER]: Refusing update from remote peer 2a06e94e44f24a35b5da6da522938c92: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:03:22.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.590860 16297 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 2a06e94e44f24a35b5da6da522938c92 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bee3c957fe9436399c523d7516cc493" member_type: VOTER last_known_addr { host: "127.15.168.188" port: 38185 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:03:22.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.613127 16293 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 2a06e94e44f24a35b5da6da522938c92. Latest consensus state: current_term: 1 leader_uuid: "2a06e94e44f24a35b5da6da522938c92" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2a06e94e44f24a35b5da6da522938c92" member_type: VOTER last_known_addr { host: "127.15.168.190" port: 33711 } } peers { permanent_uuid: "e84d2a2c5f154a8f8da59be47f9d54db" member_type: VOTER last_known_addr { host: "127.15.168.189" port: 43117 } } peers { permanent_uuid: "5bee3c957fe9436399c523d7516cc493" member_type: VOTER last_known_addr { host: "127.15.168.188" port: 38185 } } }
08:03:22.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.613423 16293 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493 [sys.catalog]: This master's current role is: FOLLOWER
08:03:22.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.616243 16322 mvcc.cc:204] Tried to move back new op lower bound from 7144538941794832384 to 7144538941658812416. Current Snapshot: MvccSnapshot[applied={T|T < 7144538941794832384}]
08:03:22.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.617254 16323 mvcc.cc:204] Tried to move back new op lower bound from 7144538941794832384 to 7144538941658812416. Current Snapshot: MvccSnapshot[applied={T|T < 7144538941794832384}]
08:03:22.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.626086 16297 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2a06e94e44f24a35b5da6da522938c92 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "2a06e94e44f24a35b5da6da522938c92" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2a06e94e44f24a35b5da6da522938c92" member_type: VOTER last_known_addr { host: "127.15.168.190" port: 33711 } } peers { permanent_uuid: "e84d2a2c5f154a8f8da59be47f9d54db" member_type: VOTER last_known_addr { host: "127.15.168.189" port: 43117 } } peers { permanent_uuid: "5bee3c957fe9436399c523d7516cc493" member_type: VOTER last_known_addr { host: "127.15.168.188" port: 38185 } } }
08:03:22.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.626412 16297 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2a06e94e44f24a35b5da6da522938c92 [sys.catalog]: This master's current role is: LEADER
08:03:22.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.628763 16293 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "2a06e94e44f24a35b5da6da522938c92" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2a06e94e44f24a35b5da6da522938c92" member_type: VOTER last_known_addr { host: "127.15.168.190" port: 33711 } } peers { permanent_uuid: "e84d2a2c5f154a8f8da59be47f9d54db" member_type: VOTER last_known_addr { host: "127.15.168.189" port: 43117 } } peers { permanent_uuid: "5bee3c957fe9436399c523d7516cc493" member_type: VOTER last_known_addr { host: "127.15.168.188" port: 38185 } } }
08:03:22.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.629024 16293 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493 [sys.catalog]: This master's current role is: FOLLOWER
08:03:22.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.634941 16321 catalog_manager.cc:1349] Generated new cluster ID: c4416adc63694b9a8ae48a3ebc360f91
08:03:22.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.635068 16321 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
08:03:22.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.660517 16202 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.168.188" port: 38185 } attempt: 5
08:03:22.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.679527 16202 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e84d2a2c5f154a8f8da59be47f9d54db: Bootstrap starting.
08:03:22.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.688082 16202 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P e84d2a2c5f154a8f8da59be47f9d54db: Neither blocks nor log segments found. Creating new log.
08:03:22.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.693491 16321 catalog_manager.cc:1372] Generated new certificate authority record
08:03:22.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.695899 16321 catalog_manager.cc:1506] Loading token signing keys...
08:03:22.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.701362 16202 log.cc:826] T 00000000000000000000000000000000 P e84d2a2c5f154a8f8da59be47f9d54db: Log is configured to *not* fsync() on all Append() calls
08:03:22.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.714882 16321 catalog_manager.cc:5954] T 00000000000000000000000000000000 P 2a06e94e44f24a35b5da6da522938c92: Generated new TSK 0
08:03:22.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.716645 16202 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e84d2a2c5f154a8f8da59be47f9d54db: No bootstrap required, opened a new log
08:03:22.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.721726 16321 catalog_manager.cc:1516] Initializing in-progress tserver states...
08:03:22.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.724570 16202 raft_consensus.cc:357] T 00000000000000000000000000000000 P e84d2a2c5f154a8f8da59be47f9d54db [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2a06e94e44f24a35b5da6da522938c92" member_type: VOTER last_known_addr { host: "127.15.168.190" port: 33711 } } peers { permanent_uuid: "e84d2a2c5f154a8f8da59be47f9d54db" member_type: VOTER last_known_addr { host: "127.15.168.189" port: 43117 } } peers { permanent_uuid: "5bee3c957fe9436399c523d7516cc493" member_type: VOTER last_known_addr { host: "127.15.168.188" port: 38185 } }
08:03:22.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.724969 16202 raft_consensus.cc:383] T 00000000000000000000000000000000 P e84d2a2c5f154a8f8da59be47f9d54db [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:22.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.725068 16202 raft_consensus.cc:738] T 00000000000000000000000000000000 P e84d2a2c5f154a8f8da59be47f9d54db [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e84d2a2c5f154a8f8da59be47f9d54db, State: Initialized, Role: FOLLOWER
08:03:22.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.725834 16202 consensus_queue.cc:260] T 00000000000000000000000000000000 P e84d2a2c5f154a8f8da59be47f9d54db [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2a06e94e44f24a35b5da6da522938c92" member_type: VOTER last_known_addr { host: "127.15.168.190" port: 33711 } } peers { permanent_uuid: "e84d2a2c5f154a8f8da59be47f9d54db" member_type: VOTER last_known_addr { host: "127.15.168.189" port: 43117 } } peers { permanent_uuid: "5bee3c957fe9436399c523d7516cc493" member_type: VOTER last_known_addr { host: "127.15.168.188" port: 38185 } }
08:03:22.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.734972 16202 sys_catalog.cc:564] T 00000000000000000000000000000000 P e84d2a2c5f154a8f8da59be47f9d54db [sys.catalog]: configured and running, proceeding with master startup.
08:03:22.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.735064 16167 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2a06e94e44f24a35b5da6da522938c92" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e84d2a2c5f154a8f8da59be47f9d54db" is_pre_election: true
08:03:22.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.735711 16167 raft_consensus.cc:2466] T 00000000000000000000000000000000 P e84d2a2c5f154a8f8da59be47f9d54db [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2a06e94e44f24a35b5da6da522938c92 in term 0.
08:03:22.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.732915 16328 sys_catalog.cc:455] T 00000000000000000000000000000000 P e84d2a2c5f154a8f8da59be47f9d54db [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2a06e94e44f24a35b5da6da522938c92" member_type: VOTER last_known_addr { host: "127.15.168.190" port: 33711 } } peers { permanent_uuid: "e84d2a2c5f154a8f8da59be47f9d54db" member_type: VOTER last_known_addr { host: "127.15.168.189" port: 43117 } } peers { permanent_uuid: "5bee3c957fe9436399c523d7516cc493" member_type: VOTER last_known_addr { host: "127.15.168.188" port: 38185 } } }
08:03:22.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.736802 16328 sys_catalog.cc:458] T 00000000000000000000000000000000 P e84d2a2c5f154a8f8da59be47f9d54db [sys.catalog]: This master's current role is: FOLLOWER
08:03:22.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.737079 16166 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2a06e94e44f24a35b5da6da522938c92" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e84d2a2c5f154a8f8da59be47f9d54db"
08:03:22.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.737370 16166 raft_consensus.cc:3058] T 00000000000000000000000000000000 P e84d2a2c5f154a8f8da59be47f9d54db [term 0 FOLLOWER]: Advancing to term 1
08:03:22.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.740819 16166 raft_consensus.cc:2466] T 00000000000000000000000000000000 P e84d2a2c5f154a8f8da59be47f9d54db [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2a06e94e44f24a35b5da6da522938c92 in term 1.
08:03:22.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.742400 16165 raft_consensus.cc:1273] T 00000000000000000000000000000000 P e84d2a2c5f154a8f8da59be47f9d54db [term 1 FOLLOWER]: Refusing update from remote peer 2a06e94e44f24a35b5da6da522938c92: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:03:22.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.749763 16319 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 2a06e94e44f24a35b5da6da522938c92 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e84d2a2c5f154a8f8da59be47f9d54db" member_type: VOTER last_known_addr { host: "127.15.168.189" port: 43117 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:03:22.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.752523 16034 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskbKfKhV/build/asan/bin/kudu
08:03:22.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbKfKhV/build/asan/bin/kudu
08:03:22.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/ts-0/wal
08:03:22.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/ts-0/data
08:03:22.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:22.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:22.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:22.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:22.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:22.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:22.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/ts-0/logs
08:03:22.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/ts-0/data/info.pb
08:03:22.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:22.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:22.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:22.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:22.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:22.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:22.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:22.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:22.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:03:22.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:22.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.168.129:0
08:03:22.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.15.168.129
08:03:22.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.168.129
08:03:22.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:22.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.15.168.190:33711,127.15.168.189:43117,127.15.168.188:38185
08:03:22.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.168.148:38479
08:03:22.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:22.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:22.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:22.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
08:03:22.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
08:03:22.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
08:03:22.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.815730 16328 sys_catalog.cc:455] T 00000000000000000000000000000000 P e84d2a2c5f154a8f8da59be47f9d54db [sys.catalog]: SysCatalogTable state changed. Reason: New leader 2a06e94e44f24a35b5da6da522938c92. Latest consensus state: current_term: 1 leader_uuid: "2a06e94e44f24a35b5da6da522938c92" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2a06e94e44f24a35b5da6da522938c92" member_type: VOTER last_known_addr { host: "127.15.168.190" port: 33711 } } peers { permanent_uuid: "e84d2a2c5f154a8f8da59be47f9d54db" member_type: VOTER last_known_addr { host: "127.15.168.189" port: 43117 } } peers { permanent_uuid: "5bee3c957fe9436399c523d7516cc493" member_type: VOTER last_known_addr { host: "127.15.168.188" port: 38185 } } }
08:03:22.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.815706 16340 sys_catalog.cc:455] T 00000000000000000000000000000000 P e84d2a2c5f154a8f8da59be47f9d54db [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "2a06e94e44f24a35b5da6da522938c92" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2a06e94e44f24a35b5da6da522938c92" member_type: VOTER last_known_addr { host: "127.15.168.190" port: 33711 } } peers { permanent_uuid: "e84d2a2c5f154a8f8da59be47f9d54db" member_type: VOTER last_known_addr { host: "127.15.168.189" port: 43117 } } peers { permanent_uuid: "5bee3c957fe9436399c523d7516cc493" member_type: VOTER last_known_addr { host: "127.15.168.188" port: 38185 } } }
08:03:22.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.816074 16328 sys_catalog.cc:458] T 00000000000000000000000000000000 P e84d2a2c5f154a8f8da59be47f9d54db [sys.catalog]: This master's current role is: FOLLOWER
08:03:22.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.816115 16340 sys_catalog.cc:458] T 00000000000000000000000000000000 P e84d2a2c5f154a8f8da59be47f9d54db [sys.catalog]: This master's current role is: FOLLOWER
08:03:22.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.838260 16345 catalog_manager.cc:1261] Loaded cluster ID: c4416adc63694b9a8ae48a3ebc360f91
08:03:22.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.838378 16345 catalog_manager.cc:1554] T 00000000000000000000000000000000 P e84d2a2c5f154a8f8da59be47f9d54db: loading cluster ID for follower catalog manager: success
08:03:22.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.842629 16345 catalog_manager.cc:1576] T 00000000000000000000000000000000 P e84d2a2c5f154a8f8da59be47f9d54db: acquiring CA information for follower catalog manager: success
08:03:22.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.845184 16345 catalog_manager.cc:1604] T 00000000000000000000000000000000 P e84d2a2c5f154a8f8da59be47f9d54db: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:03:22.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.866317 16319 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2a06e94e44f24a35b5da6da522938c92 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "2a06e94e44f24a35b5da6da522938c92" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2a06e94e44f24a35b5da6da522938c92" member_type: VOTER last_known_addr { host: "127.15.168.190" port: 33711 } } peers { permanent_uuid: "e84d2a2c5f154a8f8da59be47f9d54db" member_type: VOTER last_known_addr { host: "127.15.168.189" port: 43117 } } peers { permanent_uuid: "5bee3c957fe9436399c523d7516cc493" member_type: VOTER last_known_addr { host: "127.15.168.188" port: 38185 } } }
08:03:22.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:22.866636 16319 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2a06e94e44f24a35b5da6da522938c92 [sys.catalog]: This master's current role is: LEADER
08:03:23.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:23.266841 16334 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:03:23.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:23.267164 16334 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:23.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:23.267294 16334 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:03:23.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:23.276252 16334 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:03:23.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:23.276392 16334 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
08:03:23.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:23.276444 16334 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
08:03:23.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:23.276481 16334 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
08:03:23.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:23.276569 16334 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.15.168.129
08:03:23.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.286626 16334 tablet_server_runner.cc:78] Tablet server non-default flags:
08:03:23.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:23.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.168.148:38479
08:03:23.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:23.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:23.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/ts-0/data
08:03:23.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/ts-0/wal
08:03:23.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:23.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:23.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.168.129:0
08:03:23.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:23.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:23.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:23.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/ts-0/data/info.pb
08:03:23.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.168.129
08:03:23.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:23.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
08:03:23.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
08:03:23.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
08:03:23.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.15.168.190:33711,127.15.168.189:43117,127.15.168.188:38185
08:03:23.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:23.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:23.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:23.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:23.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:23.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.15.168.129
08:03:23.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/ts-0/logs
08:03:23.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:23.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:23.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:03:23.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:03:23.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:03:23.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 586cfc20905df35f49541b53af4442736790b0c1
08:03:23.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:03:23.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Apr 2025 07:43:21 UTC on 24a791456cd2
08:03:23.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 5550
08:03:23.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
08:03:23.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.288522 16334 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:23.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.290381 16334 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:23.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:23.303660 16352 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:23.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:23.303874 16353 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:23.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:23.305390 16355 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:23.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.305482 16334 server_base.cc:1034] running on GCE node
08:03:23.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.306484 16334 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:23.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.307824 16334 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:23.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.309105 16334 hybrid_clock.cc:648] HybridClock initialized: now 1744272203309035 us; error 41 us; skew 500 ppm
08:03:23.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.309517 16334 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:03:23.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.318064 16334 webserver.cc:466] Webserver started at http://127.15.168.129:38425/ using document root <none> and password file <none>
08:03:23.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.318949 16334 fs_manager.cc:362] Metadata directory not provided
08:03:23.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.319095 16334 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:23.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.319499 16334 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:03:23.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.322105 16334 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/ts-0/data/instance:
08:03:23.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6741eeca17394f8b8158bcb1bf194108"
08:03:23.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-04-10 08:03:23 on dist-test-slave-cb5c"
08:03:23.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.322929 16334 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/ts-0/wal/instance:
08:03:23.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6741eeca17394f8b8158bcb1bf194108"
08:03:23.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-04-10 08:03:23 on dist-test-slave-cb5c"
08:03:23.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.328835 16334 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.004s sys 0.000s
08:03:23.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.332535 16361 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:03:23.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.334250 16334 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.000s sys 0.004s
08:03:23.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.334450 16334 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/ts-0/data,/tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/ts-0/wal
08:03:23.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6741eeca17394f8b8158bcb1bf194108"
08:03:23.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-04-10 08:03:23 on dist-test-slave-cb5c"
08:03:23.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.335068 16334 fs_report.cc:389] FS layout report
08:03:23.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:23.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/ts-0/wal
08:03:23.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/ts-0/wal
08:03:23.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/ts-0/data/data
08:03:23.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:23.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:23.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:23.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:23.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:23.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:23.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:23.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:23.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:23.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:23.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:23.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.363075 16334 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:03:23.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.364334 16334 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:23.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.364684 16334 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:23.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.368328 16334 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:03:23.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.368645 16334 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.001s sys 0.000s
08:03:23.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.368888 16334 ts_tablet_manager.cc:610] Registered 0 tablets
08:03:23.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.369055 16334 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:03:23.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.484597 16307 catalog_manager.cc:1261] Loaded cluster ID: c4416adc63694b9a8ae48a3ebc360f91
08:03:23.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.484694 16307 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493: loading cluster ID for follower catalog manager: success
08:03:23.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.487344 16307 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493: acquiring CA information for follower catalog manager: success
08:03:23.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.489254 16307 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:03:23.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.506250 16334 rpc_server.cc:307] RPC server started. Bound to: 127.15.168.129:42641
08:03:23.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.506294 16485 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.15.168.129:42641 every 8 connection(s)
08:03:23.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.507787 16334 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/ts-0/data/info.pb
08:03:23.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.510448 16487 heartbeater.cc:344] Connected to a master server at 127.15.168.190:33711
08:03:23.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.511035 16487 heartbeater.cc:461] Registering TS with master...
08:03:23.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.512279 16487 heartbeater.cc:507] Master 127.15.168.190:33711 requested a full tablet report, sending...
08:03:23.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.514729 16034 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskbKfKhV/build/asan/bin/kudu as pid 16334
08:03:23.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.514921 16034 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/ts-0/wal/instance
08:03:23.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.516070 16076 ts_manager.cc:194] Registered new tserver with Master: 6741eeca17394f8b8158bcb1bf194108 (127.15.168.129:42641)
08:03:23.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.518275 16076 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.15.168.129:41917
08:03:23.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.522473 16034 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskbKfKhV/build/asan/bin/kudu
08:03:23.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbKfKhV/build/asan/bin/kudu
08:03:23.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/ts-1/wal
08:03:23.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/ts-1/data
08:03:23.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:23.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:23.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:23.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:23.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:23.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:23.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/ts-1/logs
08:03:23.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/ts-1/data/info.pb
08:03:23.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:23.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:23.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:23.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:23.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:23.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:23.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:23.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:23.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:03:23.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:23.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.168.130:0
08:03:23.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.15.168.130
08:03:23.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.168.130
08:03:23.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:23.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.15.168.190:33711,127.15.168.189:43117,127.15.168.188:38185
08:03:23.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.168.148:38479
08:03:23.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:23.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:23.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:23.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
08:03:23.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
08:03:23.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
08:03:23.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.524405 16488 heartbeater.cc:344] Connected to a master server at 127.15.168.189:43117
08:03:23.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.524713 16488 heartbeater.cc:461] Registering TS with master...
08:03:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.525239 16488 heartbeater.cc:507] Master 127.15.168.189:43117 requested a full tablet report, sending...
08:03:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.525317 16486 heartbeater.cc:344] Connected to a master server at 127.15.168.188:38185
08:03:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.525525 16486 heartbeater.cc:461] Registering TS with master...
08:03:23.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.525887 16486 heartbeater.cc:507] Master 127.15.168.188:38185 requested a full tablet report, sending...
08:03:23.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.527570 16157 ts_manager.cc:194] Registered new tserver with Master: 6741eeca17394f8b8158bcb1bf194108 (127.15.168.129:42641)
08:03:23.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.527639 16238 ts_manager.cc:194] Registered new tserver with Master: 6741eeca17394f8b8158bcb1bf194108 (127.15.168.129:42641)
08:03:23.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:23.857590 16491 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:03:23.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:23.857932 16491 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:23.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:23.858031 16491 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:03:23.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:23.866657 16491 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:03:23.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:23.866858 16491 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
08:03:23.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:23.866920 16491 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
08:03:23.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:23.866959 16491 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
08:03:23.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:23.867051 16491 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.15.168.130
08:03:23.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.877697 16491 tablet_server_runner.cc:78] Tablet server non-default flags:
08:03:23.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:23.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.168.148:38479
08:03:23.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:23.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:23.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/ts-1/data
08:03:23.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/ts-1/wal
08:03:23.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:23.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:23.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.168.130:0
08:03:23.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:23.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:23.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:23.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/ts-1/data/info.pb
08:03:23.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.168.130
08:03:23.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:23.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
08:03:23.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
08:03:23.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
08:03:23.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.15.168.190:33711,127.15.168.189:43117,127.15.168.188:38185
08:03:23.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:23.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:23.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:23.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:23.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:23.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.15.168.130
08:03:23.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/ts-1/logs
08:03:23.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:23.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:23.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:03:23.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:03:23.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:03:23.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 586cfc20905df35f49541b53af4442736790b0c1
08:03:23.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:03:23.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Apr 2025 07:43:21 UTC on 24a791456cd2
08:03:23.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 5550
08:03:23.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
08:03:23.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.879590 16491 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:23.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.881472 16491 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:23.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:23.893707 16497 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:23.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:23.893774 16496 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:23.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:23.893743 16499 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:23.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.894829 16491 server_base.cc:1034] running on GCE node
08:03:23.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.895424 16491 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:23.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.896310 16491 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:23.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.897527 16491 hybrid_clock.cc:648] HybridClock initialized: now 1744272203897468 us; error 26 us; skew 500 ppm
08:03:23.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.897823 16491 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:03:23.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.900132 16491 webserver.cc:466] Webserver started at http://127.15.168.130:33531/ using document root <none> and password file <none>
08:03:23.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.900856 16491 fs_manager.cc:362] Metadata directory not provided
08:03:23.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.900974 16491 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:23.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.901293 16491 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:03:23.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.903784 16491 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/ts-1/data/instance:
08:03:23.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f7d247a1721943c49472305b5bbab7a6"
08:03:23.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-04-10 08:03:23 on dist-test-slave-cb5c"
08:03:23.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.904592 16491 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/ts-1/wal/instance:
08:03:23.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f7d247a1721943c49472305b5bbab7a6"
08:03:23.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-04-10 08:03:23 on dist-test-slave-cb5c"
08:03:23.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.909979 16491 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.005s sys 0.001s
08:03:23.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.913548 16505 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:03:23.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.914850 16491 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
08:03:23.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.914987 16491 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/ts-1/data,/tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/ts-1/wal
08:03:23.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f7d247a1721943c49472305b5bbab7a6"
08:03:23.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-04-10 08:03:23 on dist-test-slave-cb5c"
08:03:23.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.915184 16491 fs_report.cc:389] FS layout report
08:03:23.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:23.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/ts-1/wal
08:03:23.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/ts-1/wal
08:03:23.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/ts-1/data/data
08:03:23.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:23.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:23.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:23.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:23.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:23.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:23.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:23.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:23.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:23.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:23.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:23.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.933372 16491 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:03:23.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.934195 16491 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:23.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.934444 16491 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:23.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.937624 16491 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:03:23.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.937884 16491 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
08:03:23.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.938177 16491 ts_tablet_manager.cc:610] Registered 0 tablets
08:03:23.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.938280 16491 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:03:23.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.998005 16491 rpc_server.cc:307] RPC server started. Bound to: 127.15.168.130:45579
08:03:23.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.998078 16626 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.15.168.130:45579 every 8 connection(s)
08:03:23.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:23.999359 16491 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/ts-1/data/info.pb
08:03:24.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.000047 16034 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskbKfKhV/build/asan/bin/kudu as pid 16491
08:03:24.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.000233 16034 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/ts-1/wal/instance
08:03:24.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.002372 16628 heartbeater.cc:344] Connected to a master server at 127.15.168.190:33711
08:03:24.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.002925 16628 heartbeater.cc:461] Registering TS with master...
08:03:24.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.004928 16628 heartbeater.cc:507] Master 127.15.168.190:33711 requested a full tablet report, sending...
08:03:24.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.006436 16076 ts_manager.cc:194] Registered new tserver with Master: f7d247a1721943c49472305b5bbab7a6 (127.15.168.130:45579)
08:03:24.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.007612 16076 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.15.168.130:41439
08:03:24.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.011045 16034 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskbKfKhV/build/asan/bin/kudu
08:03:24.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbKfKhV/build/asan/bin/kudu
08:03:24.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/ts-2/wal
08:03:24.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/ts-2/data
08:03:24.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:24.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:24.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:24.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:24.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:24.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:24.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/ts-2/logs
08:03:24.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/ts-2/data/info.pb
08:03:24.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:24.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:24.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:24.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:24.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:24.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:24.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:24.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:24.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:03:24.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:24.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.168.131:0
08:03:24.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.15.168.131
08:03:24.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.168.131
08:03:24.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:24.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.15.168.190:33711,127.15.168.189:43117,127.15.168.188:38185
08:03:24.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.168.148:38479
08:03:24.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:24.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:24.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:24.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
08:03:24.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
08:03:24.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
08:03:24.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.018227 16627 heartbeater.cc:344] Connected to a master server at 127.15.168.188:38185
08:03:24.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.018417 16627 heartbeater.cc:461] Registering TS with master...
08:03:24.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.018775 16627 heartbeater.cc:507] Master 127.15.168.188:38185 requested a full tablet report, sending...
08:03:24.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.019830 16238 ts_manager.cc:194] Registered new tserver with Master: f7d247a1721943c49472305b5bbab7a6 (127.15.168.130:45579)
08:03:24.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.024515 16629 heartbeater.cc:344] Connected to a master server at 127.15.168.189:43117
08:03:24.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.024695 16629 heartbeater.cc:461] Registering TS with master...
08:03:24.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.024997 16629 heartbeater.cc:507] Master 127.15.168.189:43117 requested a full tablet report, sending...
08:03:24.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.026106 16157 ts_manager.cc:194] Registered new tserver with Master: f7d247a1721943c49472305b5bbab7a6 (127.15.168.130:45579)
08:03:24.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:24.344439 16632 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:03:24.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:24.344759 16632 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:24.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:24.344853 16632 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:03:24.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:24.352941 16632 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:03:24.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:24.353133 16632 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
08:03:24.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:24.353199 16632 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
08:03:24.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:24.353240 16632 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
08:03:24.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:24.353381 16632 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.15.168.131
08:03:24.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.363001 16632 tablet_server_runner.cc:78] Tablet server non-default flags:
08:03:24.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:24.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.168.148:38479
08:03:24.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:24.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:24.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/ts-2/data
08:03:24.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/ts-2/wal
08:03:24.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:24.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:24.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.168.131:0
08:03:24.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:24.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:24.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:24.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/ts-2/data/info.pb
08:03:24.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.168.131
08:03:24.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:24.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
08:03:24.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
08:03:24.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
08:03:24.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.15.168.190:33711,127.15.168.189:43117,127.15.168.188:38185
08:03:24.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:24.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:24.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:24.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:24.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:24.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.15.168.131
08:03:24.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/ts-2/logs
08:03:24.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:24.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:24.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:03:24.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:03:24.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:03:24.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 586cfc20905df35f49541b53af4442736790b0c1
08:03:24.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:03:24.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Apr 2025 07:43:21 UTC on 24a791456cd2
08:03:24.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 5550
08:03:24.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
08:03:24.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.364672 16632 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:24.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.366998 16632 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:24.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:24.379406 16638 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:24.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:24.379380 16639 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:24.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:24.379501 16641 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:24.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.380192 16632 server_base.cc:1034] running on GCE node
08:03:24.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.380889 16632 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:24.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.381935 16632 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:24.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.383203 16632 hybrid_clock.cc:648] HybridClock initialized: now 1744272204383105 us; error 36 us; skew 500 ppm
08:03:24.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.383538 16632 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:03:24.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.385952 16632 webserver.cc:466] Webserver started at http://127.15.168.131:44709/ using document root <none> and password file <none>
08:03:24.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.386782 16632 fs_manager.cc:362] Metadata directory not provided
08:03:24.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.386917 16632 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:24.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.387301 16632 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:03:24.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.389880 16632 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/ts-2/data/instance:
08:03:24.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "82c7f3b9c7774a5281eae28fb7e64216"
08:03:24.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-04-10 08:03:24 on dist-test-slave-cb5c"
08:03:24.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.390746 16632 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/ts-2/wal/instance:
08:03:24.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "82c7f3b9c7774a5281eae28fb7e64216"
08:03:24.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-04-10 08:03:24 on dist-test-slave-cb5c"
08:03:24.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.396394 16632 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.002s sys 0.001s
08:03:24.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.400097 16647 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:03:24.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.401697 16632 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
08:03:24.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.401846 16632 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/ts-2/data,/tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/ts-2/wal
08:03:24.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "82c7f3b9c7774a5281eae28fb7e64216"
08:03:24.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-04-10 08:03:24 on dist-test-slave-cb5c"
08:03:24.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.402009 16632 fs_report.cc:389] FS layout report
08:03:24.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:24.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/ts-2/wal
08:03:24.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/ts-2/wal
08:03:24.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/ts-2/data/data
08:03:24.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:24.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:24.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:24.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:24.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:24.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:24.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:24.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:24.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:24.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:24.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:24.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.429260 16632 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:03:24.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.430119 16632 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:24.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.430392 16632 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:24.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.433600 16632 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:03:24.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.433750 16632 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
08:03:24.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.433884 16632 ts_tablet_manager.cc:610] Registered 0 tablets
08:03:24.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.433956 16632 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:03:24.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.497924 16632 rpc_server.cc:307] RPC server started. Bound to: 127.15.168.131:40597
08:03:24.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.497973 16768 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.15.168.131:40597 every 8 connection(s)
08:03:24.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.499377 16632 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/ts-2/data/info.pb
08:03:24.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.502635 16034 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskbKfKhV/build/asan/bin/kudu as pid 16632
08:03:24.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.502820 16034 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskbKfKhV/test-tmp/mini-kudu-cluster7040418216636735382/ts-2/wal/instance
08:03:24.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.502650 16770 heartbeater.cc:344] Connected to a master server at 127.15.168.190:33711
08:03:24.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.503235 16770 heartbeater.cc:461] Registering TS with master...
08:03:24.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.504688 16770 heartbeater.cc:507] Master 127.15.168.190:33711 requested a full tablet report, sending...
08:03:24.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.506690 16076 ts_manager.cc:194] Registered new tserver with Master: 82c7f3b9c7774a5281eae28fb7e64216 (127.15.168.131:40597)
08:03:24.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.507877 16076 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.15.168.131:45467
08:03:24.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.513255 16771 heartbeater.cc:344] Connected to a master server at 127.15.168.189:43117
08:03:24.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.513422 16771 heartbeater.cc:461] Registering TS with master...
08:03:24.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.513845 16771 heartbeater.cc:507] Master 127.15.168.189:43117 requested a full tablet report, sending...
08:03:24.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.514842 16769 heartbeater.cc:344] Connected to a master server at 127.15.168.188:38185
08:03:24.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.514982 16769 heartbeater.cc:461] Registering TS with master...
08:03:24.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.515233 16156 ts_manager.cc:194] Registered new tserver with Master: 82c7f3b9c7774a5281eae28fb7e64216 (127.15.168.131:40597)
08:03:24.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.515300 16769 heartbeater.cc:507] Master 127.15.168.188:38185 requested a full tablet report, sending...
08:03:24.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.516340 16238 ts_manager.cc:194] Registered new tserver with Master: 82c7f3b9c7774a5281eae28fb7e64216 (127.15.168.131:40597)
08:03:24.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.523286 16487 heartbeater.cc:499] Master 127.15.168.190:33711 was elected leader, sending a full tablet report...
08:03:24.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.529948 16034 external_mini_cluster.cc:934] 3 TS(s) registered with all masters
08:03:24.531 [DEBUG - main] (MiniKuduCluster.java:195) Response:
08:03:24.531 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
08:03:24.532 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.15.168.190"
port: 33711
}
bound_http_address {
host: "127.15.168.190"
port: 44023
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.15.168.189"
port: 43117
}
bound_http_address {
host: "127.15.168.189"
port: 43431
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.15.168.188"
port: 38185
}
bound_http_address {
host: "127.15.168.188"
port: 36971
}
}
}
08:03:24.533 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
08:03:24.533 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.15.168.129"
port: 42641
}
bound_http_address {
host: "127.15.168.129"
port: 38425
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.15.168.130"
port: 45579
}
bound_http_address {
host: "127.15.168.130"
port: 33531
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.15.168.131"
port: 40597
}
bound_http_address {
host: "127.15.168.131"
port: 44709
}
}
}
08:03:24.533 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
08:03:24.535 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
08:03:24.537 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.15.168.188:38185(127.15.168.188:38185)] connecting to peer
08:03:24.537 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.15.168.190:33711(127.15.168.190:33711)] connecting to peer
08:03:24.538 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.15.168.189:43117(127.15.168.189:43117)] connecting to peer
08:03:24.538 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.15.168.188:38185(127.15.168.188:38185)] Successfully connected to peer
08:03:24.539 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.15.168.190:33711(127.15.168.190:33711)] Successfully connected to peer
08:03:24.541 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.15.168.188
08:03:24.541 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.15.168.190
08:03:24.546 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.15.168.189:43117(127.15.168.189:43117)] Successfully connected to peer
08:03:24.554 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.15.168.189
08:03:24.556 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x4b66c0e7, L:/127.0.0.1:50658 - R:/127.15.168.190:33711] using SASL/PLAIN
08:03:24.558 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
08:03:24.558 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=22), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.188:38185, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.190:33711, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.189:43117, [21ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.190:33711: OK, deferred=null) after lookup
08:03:24.559 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x5cfc4b76, L:/127.0.0.1:36328 - R:/127.15.168.188:38185] using SASL/PLAIN
08:03:24.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.560912 16076 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:50658:
08:03:24.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_manager_fallback"
08:03:24.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
08:03:24.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:24.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
08:03:24.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
08:03:24.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:03:24.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:24.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:24.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:24.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
08:03:24.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:24.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:24.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column1_i"
08:03:24.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
08:03:24.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
08:03:24.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:24.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:24.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:24.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
08:03:24.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:24.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:24.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column2_i"
08:03:24.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
08:03:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
08:03:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
08:03:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column3_s"
08:03:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
08:03:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
08:03:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
08:03:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: DICT_ENCODING
08:03:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: LZ4
08:03:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 4096
08:03:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
08:03:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column4_b"
08:03:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: BOOL
08:03:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
08:03:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
08:03:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
08:03:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) hash_schema {
08:03:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
08:03:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_buckets: 2
08:03:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) seed: 0
08:03:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:24.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:24.562899 16076 catalog_manager.cc:6943] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table txn_manager_fallback in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
08:03:24.566 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xedb6a2fd, L:/127.0.0.1:38866 - R:/127.15.168.189:43117] using SASL/PLAIN
08:03:24.587 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table c38065584d484ecb87535dca2fe900a1
08:03:24.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.625141 16420 tablet_service.cc:1467] Processing CreateTablet for tablet e904ce805e75413abce8e621262820b8 (DEFAULT_TABLE table=txn_manager_fallback [id=c38065584d484ecb87535dca2fe900a1]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
08:03:24.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.625195 16421 tablet_service.cc:1467] Processing CreateTablet for tablet fac3dfebf08c4f418ff9fa5397d50084 (DEFAULT_TABLE table=txn_manager_fallback [id=c38065584d484ecb87535dca2fe900a1]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
08:03:24.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.626816 16561 tablet_service.cc:1467] Processing CreateTablet for tablet e904ce805e75413abce8e621262820b8 (DEFAULT_TABLE table=txn_manager_fallback [id=c38065584d484ecb87535dca2fe900a1]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
08:03:24.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.627051 16420 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet e904ce805e75413abce8e621262820b8. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:24.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.627409 16562 tablet_service.cc:1467] Processing CreateTablet for tablet fac3dfebf08c4f418ff9fa5397d50084 (DEFAULT_TABLE table=txn_manager_fallback [id=c38065584d484ecb87535dca2fe900a1]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
08:03:24.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.628569 16562 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet fac3dfebf08c4f418ff9fa5397d50084. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:24.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.629067 16561 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet e904ce805e75413abce8e621262820b8. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:24.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.629402 16700 tablet_service.cc:1467] Processing CreateTablet for tablet e904ce805e75413abce8e621262820b8 (DEFAULT_TABLE table=txn_manager_fallback [id=c38065584d484ecb87535dca2fe900a1]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
08:03:24.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.629431 16701 tablet_service.cc:1467] Processing CreateTablet for tablet fac3dfebf08c4f418ff9fa5397d50084 (DEFAULT_TABLE table=txn_manager_fallback [id=c38065584d484ecb87535dca2fe900a1]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
08:03:24.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.631003 16700 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet e904ce805e75413abce8e621262820b8. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:24.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.631439 16421 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet fac3dfebf08c4f418ff9fa5397d50084. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:24.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.632395 16701 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet fac3dfebf08c4f418ff9fa5397d50084. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:24.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.648995 16794 tablet_bootstrap.cc:492] T fac3dfebf08c4f418ff9fa5397d50084 P 6741eeca17394f8b8158bcb1bf194108: Bootstrap starting.
08:03:24.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.652143 16795 tablet_bootstrap.cc:492] T e904ce805e75413abce8e621262820b8 P f7d247a1721943c49472305b5bbab7a6: Bootstrap starting.
08:03:24.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.655825 16794 tablet_bootstrap.cc:654] T fac3dfebf08c4f418ff9fa5397d50084 P 6741eeca17394f8b8158bcb1bf194108: Neither blocks nor log segments found. Creating new log.
08:03:24.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.656911 16795 tablet_bootstrap.cc:654] T e904ce805e75413abce8e621262820b8 P f7d247a1721943c49472305b5bbab7a6: Neither blocks nor log segments found. Creating new log.
08:03:24.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.656931 16796 tablet_bootstrap.cc:492] T e904ce805e75413abce8e621262820b8 P 82c7f3b9c7774a5281eae28fb7e64216: Bootstrap starting.
08:03:24.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.657301 16794 log.cc:826] T fac3dfebf08c4f418ff9fa5397d50084 P 6741eeca17394f8b8158bcb1bf194108: Log is configured to *not* fsync() on all Append() calls
08:03:24.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.658998 16795 log.cc:826] T e904ce805e75413abce8e621262820b8 P f7d247a1721943c49472305b5bbab7a6: Log is configured to *not* fsync() on all Append() calls
08:03:24.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.660712 16796 tablet_bootstrap.cc:654] T e904ce805e75413abce8e621262820b8 P 82c7f3b9c7774a5281eae28fb7e64216: Neither blocks nor log segments found. Creating new log.
08:03:24.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.661266 16794 tablet_bootstrap.cc:492] T fac3dfebf08c4f418ff9fa5397d50084 P 6741eeca17394f8b8158bcb1bf194108: No bootstrap required, opened a new log
08:03:24.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.661609 16794 ts_tablet_manager.cc:1397] T fac3dfebf08c4f418ff9fa5397d50084 P 6741eeca17394f8b8158bcb1bf194108: Time spent bootstrapping tablet: real 0.013s user 0.008s sys 0.000s
08:03:24.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.661859 16796 log.cc:826] T e904ce805e75413abce8e621262820b8 P 82c7f3b9c7774a5281eae28fb7e64216: Log is configured to *not* fsync() on all Append() calls
08:03:24.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.661947 16795 tablet_bootstrap.cc:492] T e904ce805e75413abce8e621262820b8 P f7d247a1721943c49472305b5bbab7a6: No bootstrap required, opened a new log
08:03:24.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.662228 16795 ts_tablet_manager.cc:1397] T e904ce805e75413abce8e621262820b8 P f7d247a1721943c49472305b5bbab7a6: Time spent bootstrapping tablet: real 0.010s user 0.006s sys 0.000s
08:03:24.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.665202 16796 tablet_bootstrap.cc:492] T e904ce805e75413abce8e621262820b8 P 82c7f3b9c7774a5281eae28fb7e64216: No bootstrap required, opened a new log
08:03:24.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.665483 16796 ts_tablet_manager.cc:1397] T e904ce805e75413abce8e621262820b8 P 82c7f3b9c7774a5281eae28fb7e64216: Time spent bootstrapping tablet: real 0.009s user 0.003s sys 0.004s
08:03:24.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.669189 16795 raft_consensus.cc:357] T e904ce805e75413abce8e621262820b8 P f7d247a1721943c49472305b5bbab7a6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6741eeca17394f8b8158bcb1bf194108" member_type: VOTER last_known_addr { host: "127.15.168.129" port: 42641 } } peers { permanent_uuid: "f7d247a1721943c49472305b5bbab7a6" member_type: VOTER last_known_addr { host: "127.15.168.130" port: 45579 } } peers { permanent_uuid: "82c7f3b9c7774a5281eae28fb7e64216" member_type: VOTER last_known_addr { host: "127.15.168.131" port: 40597 } }
08:03:24.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.669189 16794 raft_consensus.cc:357] T fac3dfebf08c4f418ff9fa5397d50084 P 6741eeca17394f8b8158bcb1bf194108 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82c7f3b9c7774a5281eae28fb7e64216" member_type: VOTER last_known_addr { host: "127.15.168.131" port: 40597 } } peers { permanent_uuid: "6741eeca17394f8b8158bcb1bf194108" member_type: VOTER last_known_addr { host: "127.15.168.129" port: 42641 } } peers { permanent_uuid: "f7d247a1721943c49472305b5bbab7a6" member_type: VOTER last_known_addr { host: "127.15.168.130" port: 45579 } }
08:03:24.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.669634 16794 raft_consensus.cc:383] T fac3dfebf08c4f418ff9fa5397d50084 P 6741eeca17394f8b8158bcb1bf194108 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:24.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.669634 16795 raft_consensus.cc:383] T e904ce805e75413abce8e621262820b8 P f7d247a1721943c49472305b5bbab7a6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:24.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.669759 16795 raft_consensus.cc:738] T e904ce805e75413abce8e621262820b8 P f7d247a1721943c49472305b5bbab7a6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f7d247a1721943c49472305b5bbab7a6, State: Initialized, Role: FOLLOWER
08:03:24.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.669761 16794 raft_consensus.cc:738] T fac3dfebf08c4f418ff9fa5397d50084 P 6741eeca17394f8b8158bcb1bf194108 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6741eeca17394f8b8158bcb1bf194108, State: Initialized, Role: FOLLOWER
08:03:24.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.670451 16795 consensus_queue.cc:260] T e904ce805e75413abce8e621262820b8 P f7d247a1721943c49472305b5bbab7a6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6741eeca17394f8b8158bcb1bf194108" member_type: VOTER last_known_addr { host: "127.15.168.129" port: 42641 } } peers { permanent_uuid: "f7d247a1721943c49472305b5bbab7a6" member_type: VOTER last_known_addr { host: "127.15.168.130" port: 45579 } } peers { permanent_uuid: "82c7f3b9c7774a5281eae28fb7e64216" member_type: VOTER last_known_addr { host: "127.15.168.131" port: 40597 } }
08:03:24.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.670447 16794 consensus_queue.cc:260] T fac3dfebf08c4f418ff9fa5397d50084 P 6741eeca17394f8b8158bcb1bf194108 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82c7f3b9c7774a5281eae28fb7e64216" member_type: VOTER last_known_addr { host: "127.15.168.131" port: 40597 } } peers { permanent_uuid: "6741eeca17394f8b8158bcb1bf194108" member_type: VOTER last_known_addr { host: "127.15.168.129" port: 42641 } } peers { permanent_uuid: "f7d247a1721943c49472305b5bbab7a6" member_type: VOTER last_known_addr { host: "127.15.168.130" port: 45579 } }
08:03:24.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.671027 16796 raft_consensus.cc:357] T e904ce805e75413abce8e621262820b8 P 82c7f3b9c7774a5281eae28fb7e64216 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6741eeca17394f8b8158bcb1bf194108" member_type: VOTER last_known_addr { host: "127.15.168.129" port: 42641 } } peers { permanent_uuid: "f7d247a1721943c49472305b5bbab7a6" member_type: VOTER last_known_addr { host: "127.15.168.130" port: 45579 } } peers { permanent_uuid: "82c7f3b9c7774a5281eae28fb7e64216" member_type: VOTER last_known_addr { host: "127.15.168.131" port: 40597 } }
08:03:24.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.671437 16796 raft_consensus.cc:383] T e904ce805e75413abce8e621262820b8 P 82c7f3b9c7774a5281eae28fb7e64216 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:24.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.671555 16796 raft_consensus.cc:738] T e904ce805e75413abce8e621262820b8 P 82c7f3b9c7774a5281eae28fb7e64216 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82c7f3b9c7774a5281eae28fb7e64216, State: Initialized, Role: FOLLOWER
08:03:24.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.672382 16794 ts_tablet_manager.cc:1428] T fac3dfebf08c4f418ff9fa5397d50084 P 6741eeca17394f8b8158bcb1bf194108: Time spent starting tablet: real 0.011s user 0.012s sys 0.000s
08:03:24.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.672466 16628 heartbeater.cc:499] Master 127.15.168.190:33711 was elected leader, sending a full tablet report...
08:03:24.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.672924 16795 ts_tablet_manager.cc:1428] T e904ce805e75413abce8e621262820b8 P f7d247a1721943c49472305b5bbab7a6: Time spent starting tablet: real 0.011s user 0.012s sys 0.000s
08:03:24.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.673494 16795 tablet_bootstrap.cc:492] T fac3dfebf08c4f418ff9fa5397d50084 P f7d247a1721943c49472305b5bbab7a6: Bootstrap starting.
08:03:24.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.675547 16795 tablet_bootstrap.cc:654] T fac3dfebf08c4f418ff9fa5397d50084 P f7d247a1721943c49472305b5bbab7a6: Neither blocks nor log segments found. Creating new log.
08:03:24.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.680022 16794 tablet_bootstrap.cc:492] T e904ce805e75413abce8e621262820b8 P 6741eeca17394f8b8158bcb1bf194108: Bootstrap starting.
08:03:24.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.683061 16794 tablet_bootstrap.cc:654] T e904ce805e75413abce8e621262820b8 P 6741eeca17394f8b8158bcb1bf194108: Neither blocks nor log segments found. Creating new log.
08:03:24.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.672230 16796 consensus_queue.cc:260] T e904ce805e75413abce8e621262820b8 P 82c7f3b9c7774a5281eae28fb7e64216 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6741eeca17394f8b8158bcb1bf194108" member_type: VOTER last_known_addr { host: "127.15.168.129" port: 42641 } } peers { permanent_uuid: "f7d247a1721943c49472305b5bbab7a6" member_type: VOTER last_known_addr { host: "127.15.168.130" port: 45579 } } peers { permanent_uuid: "82c7f3b9c7774a5281eae28fb7e64216" member_type: VOTER last_known_addr { host: "127.15.168.131" port: 40597 } }
08:03:24.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.692835 16795 tablet_bootstrap.cc:492] T fac3dfebf08c4f418ff9fa5397d50084 P f7d247a1721943c49472305b5bbab7a6: No bootstrap required, opened a new log
08:03:24.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.693070 16795 ts_tablet_manager.cc:1397] T fac3dfebf08c4f418ff9fa5397d50084 P f7d247a1721943c49472305b5bbab7a6: Time spent bootstrapping tablet: real 0.020s user 0.014s sys 0.004s
08:03:24.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.694227 16795 raft_consensus.cc:357] T fac3dfebf08c4f418ff9fa5397d50084 P f7d247a1721943c49472305b5bbab7a6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82c7f3b9c7774a5281eae28fb7e64216" member_type: VOTER last_known_addr { host: "127.15.168.131" port: 40597 } } peers { permanent_uuid: "6741eeca17394f8b8158bcb1bf194108" member_type: VOTER last_known_addr { host: "127.15.168.129" port: 42641 } } peers { permanent_uuid: "f7d247a1721943c49472305b5bbab7a6" member_type: VOTER last_known_addr { host: "127.15.168.130" port: 45579 } }
08:03:24.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.694519 16795 raft_consensus.cc:383] T fac3dfebf08c4f418ff9fa5397d50084 P f7d247a1721943c49472305b5bbab7a6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:24.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.694598 16795 raft_consensus.cc:738] T fac3dfebf08c4f418ff9fa5397d50084 P f7d247a1721943c49472305b5bbab7a6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f7d247a1721943c49472305b5bbab7a6, State: Initialized, Role: FOLLOWER
08:03:24.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.694864 16795 consensus_queue.cc:260] T fac3dfebf08c4f418ff9fa5397d50084 P f7d247a1721943c49472305b5bbab7a6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82c7f3b9c7774a5281eae28fb7e64216" member_type: VOTER last_known_addr { host: "127.15.168.131" port: 40597 } } peers { permanent_uuid: "6741eeca17394f8b8158bcb1bf194108" member_type: VOTER last_known_addr { host: "127.15.168.129" port: 42641 } } peers { permanent_uuid: "f7d247a1721943c49472305b5bbab7a6" member_type: VOTER last_known_addr { host: "127.15.168.130" port: 45579 } }
08:03:24.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.695438 16795 ts_tablet_manager.cc:1428] T fac3dfebf08c4f418ff9fa5397d50084 P f7d247a1721943c49472305b5bbab7a6: Time spent starting tablet: real 0.002s user 0.003s sys 0.000s
08:03:24.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.706894 16796 ts_tablet_manager.cc:1428] T e904ce805e75413abce8e621262820b8 P 82c7f3b9c7774a5281eae28fb7e64216: Time spent starting tablet: real 0.041s user 0.016s sys 0.012s
08:03:24.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.707324 16770 heartbeater.cc:499] Master 127.15.168.190:33711 was elected leader, sending a full tablet report...
08:03:24.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.711822 16796 tablet_bootstrap.cc:492] T fac3dfebf08c4f418ff9fa5397d50084 P 82c7f3b9c7774a5281eae28fb7e64216: Bootstrap starting.
08:03:24.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.714027 16796 tablet_bootstrap.cc:654] T fac3dfebf08c4f418ff9fa5397d50084 P 82c7f3b9c7774a5281eae28fb7e64216: Neither blocks nor log segments found. Creating new log.
08:03:24.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.721617 16796 tablet_bootstrap.cc:492] T fac3dfebf08c4f418ff9fa5397d50084 P 82c7f3b9c7774a5281eae28fb7e64216: No bootstrap required, opened a new log
08:03:24.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.721675 16794 tablet_bootstrap.cc:492] T e904ce805e75413abce8e621262820b8 P 6741eeca17394f8b8158bcb1bf194108: No bootstrap required, opened a new log
08:03:24.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.721807 16796 ts_tablet_manager.cc:1397] T fac3dfebf08c4f418ff9fa5397d50084 P 82c7f3b9c7774a5281eae28fb7e64216: Time spent bootstrapping tablet: real 0.010s user 0.009s sys 0.000s
08:03:24.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.721853 16794 ts_tablet_manager.cc:1397] T e904ce805e75413abce8e621262820b8 P 6741eeca17394f8b8158bcb1bf194108: Time spent bootstrapping tablet: real 0.042s user 0.023s sys 0.010s
08:03:24.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.722600 16796 raft_consensus.cc:357] T fac3dfebf08c4f418ff9fa5397d50084 P 82c7f3b9c7774a5281eae28fb7e64216 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82c7f3b9c7774a5281eae28fb7e64216" member_type: VOTER last_known_addr { host: "127.15.168.131" port: 40597 } } peers { permanent_uuid: "6741eeca17394f8b8158bcb1bf194108" member_type: VOTER last_known_addr { host: "127.15.168.129" port: 42641 } } peers { permanent_uuid: "f7d247a1721943c49472305b5bbab7a6" member_type: VOTER last_known_addr { host: "127.15.168.130" port: 45579 } }
08:03:24.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.722658 16794 raft_consensus.cc:357] T e904ce805e75413abce8e621262820b8 P 6741eeca17394f8b8158bcb1bf194108 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6741eeca17394f8b8158bcb1bf194108" member_type: VOTER last_known_addr { host: "127.15.168.129" port: 42641 } } peers { permanent_uuid: "f7d247a1721943c49472305b5bbab7a6" member_type: VOTER last_known_addr { host: "127.15.168.130" port: 45579 } } peers { permanent_uuid: "82c7f3b9c7774a5281eae28fb7e64216" member_type: VOTER last_known_addr { host: "127.15.168.131" port: 40597 } }
08:03:24.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.722785 16796 raft_consensus.cc:383] T fac3dfebf08c4f418ff9fa5397d50084 P 82c7f3b9c7774a5281eae28fb7e64216 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:24.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.722857 16794 raft_consensus.cc:383] T e904ce805e75413abce8e621262820b8 P 6741eeca17394f8b8158bcb1bf194108 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:24.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.722857 16796 raft_consensus.cc:738] T fac3dfebf08c4f418ff9fa5397d50084 P 82c7f3b9c7774a5281eae28fb7e64216 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82c7f3b9c7774a5281eae28fb7e64216, State: Initialized, Role: FOLLOWER
08:03:24.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.722922 16794 raft_consensus.cc:738] T e904ce805e75413abce8e621262820b8 P 6741eeca17394f8b8158bcb1bf194108 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6741eeca17394f8b8158bcb1bf194108, State: Initialized, Role: FOLLOWER
08:03:24.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.723263 16796 consensus_queue.cc:260] T fac3dfebf08c4f418ff9fa5397d50084 P 82c7f3b9c7774a5281eae28fb7e64216 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82c7f3b9c7774a5281eae28fb7e64216" member_type: VOTER last_known_addr { host: "127.15.168.131" port: 40597 } } peers { permanent_uuid: "6741eeca17394f8b8158bcb1bf194108" member_type: VOTER last_known_addr { host: "127.15.168.129" port: 42641 } } peers { permanent_uuid: "f7d247a1721943c49472305b5bbab7a6" member_type: VOTER last_known_addr { host: "127.15.168.130" port: 45579 } }
08:03:24.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.723265 16794 consensus_queue.cc:260] T e904ce805e75413abce8e621262820b8 P 6741eeca17394f8b8158bcb1bf194108 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6741eeca17394f8b8158bcb1bf194108" member_type: VOTER last_known_addr { host: "127.15.168.129" port: 42641 } } peers { permanent_uuid: "f7d247a1721943c49472305b5bbab7a6" member_type: VOTER last_known_addr { host: "127.15.168.130" port: 45579 } } peers { permanent_uuid: "82c7f3b9c7774a5281eae28fb7e64216" member_type: VOTER last_known_addr { host: "127.15.168.131" port: 40597 } }
08:03:24.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.723755 16794 ts_tablet_manager.cc:1428] T e904ce805e75413abce8e621262820b8 P 6741eeca17394f8b8158bcb1bf194108: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
08:03:24.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.726814 16796 ts_tablet_manager.cc:1428] T fac3dfebf08c4f418ff9fa5397d50084 P 82c7f3b9c7774a5281eae28fb7e64216: Time spent starting tablet: real 0.005s user 0.002s sys 0.000s
08:03:24.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:24.752844 16772 tablet.cc:2367] T e904ce805e75413abce8e621262820b8 P 82c7f3b9c7774a5281eae28fb7e64216: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:03:24.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:24.761765 16489 tablet.cc:2367] T e904ce805e75413abce8e621262820b8 P 6741eeca17394f8b8158bcb1bf194108: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:03:24.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:24.767680 16630 tablet.cc:2367] T e904ce805e75413abce8e621262820b8 P f7d247a1721943c49472305b5bbab7a6: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:03:24.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.845474 16804 raft_consensus.cc:491] T e904ce805e75413abce8e621262820b8 P 82c7f3b9c7774a5281eae28fb7e64216 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:03:24.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.845767 16804 raft_consensus.cc:513] T e904ce805e75413abce8e621262820b8 P 82c7f3b9c7774a5281eae28fb7e64216 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6741eeca17394f8b8158bcb1bf194108" member_type: VOTER last_known_addr { host: "127.15.168.129" port: 42641 } } peers { permanent_uuid: "f7d247a1721943c49472305b5bbab7a6" member_type: VOTER last_known_addr { host: "127.15.168.130" port: 45579 } } peers { permanent_uuid: "82c7f3b9c7774a5281eae28fb7e64216" member_type: VOTER last_known_addr { host: "127.15.168.131" port: 40597 } }
08:03:24.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.847622 16804 leader_election.cc:290] T e904ce805e75413abce8e621262820b8 P 82c7f3b9c7774a5281eae28fb7e64216 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6741eeca17394f8b8158bcb1bf194108 (127.15.168.129:42641), f7d247a1721943c49472305b5bbab7a6 (127.15.168.130:45579)
08:03:24.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.853941 16441 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e904ce805e75413abce8e621262820b8" candidate_uuid: "82c7f3b9c7774a5281eae28fb7e64216" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6741eeca17394f8b8158bcb1bf194108" is_pre_election: true
08:03:24.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.853941 16582 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e904ce805e75413abce8e621262820b8" candidate_uuid: "82c7f3b9c7774a5281eae28fb7e64216" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f7d247a1721943c49472305b5bbab7a6" is_pre_election: true
08:03:24.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.854561 16582 raft_consensus.cc:2466] T e904ce805e75413abce8e621262820b8 P f7d247a1721943c49472305b5bbab7a6 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82c7f3b9c7774a5281eae28fb7e64216 in term 0.
08:03:24.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.854559 16441 raft_consensus.cc:2466] T e904ce805e75413abce8e621262820b8 P 6741eeca17394f8b8158bcb1bf194108 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82c7f3b9c7774a5281eae28fb7e64216 in term 0.
08:03:24.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.855203 16648 leader_election.cc:304] T e904ce805e75413abce8e621262820b8 P 82c7f3b9c7774a5281eae28fb7e64216 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 82c7f3b9c7774a5281eae28fb7e64216, f7d247a1721943c49472305b5bbab7a6; no voters:
08:03:24.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.855557 16804 raft_consensus.cc:2802] T e904ce805e75413abce8e621262820b8 P 82c7f3b9c7774a5281eae28fb7e64216 [term 0 FOLLOWER]: Leader pre-election won for term 1
08:03:24.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.855723 16804 raft_consensus.cc:491] T e904ce805e75413abce8e621262820b8 P 82c7f3b9c7774a5281eae28fb7e64216 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:03:24.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.855835 16804 raft_consensus.cc:3058] T e904ce805e75413abce8e621262820b8 P 82c7f3b9c7774a5281eae28fb7e64216 [term 0 FOLLOWER]: Advancing to term 1
08:03:24.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.857858 16804 raft_consensus.cc:513] T e904ce805e75413abce8e621262820b8 P 82c7f3b9c7774a5281eae28fb7e64216 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6741eeca17394f8b8158bcb1bf194108" member_type: VOTER last_known_addr { host: "127.15.168.129" port: 42641 } } peers { permanent_uuid: "f7d247a1721943c49472305b5bbab7a6" member_type: VOTER last_known_addr { host: "127.15.168.130" port: 45579 } } peers { permanent_uuid: "82c7f3b9c7774a5281eae28fb7e64216" member_type: VOTER last_known_addr { host: "127.15.168.131" port: 40597 } }
08:03:24.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.858423 16804 leader_election.cc:290] T e904ce805e75413abce8e621262820b8 P 82c7f3b9c7774a5281eae28fb7e64216 [CANDIDATE]: Term 1 election: Requested vote from peers 6741eeca17394f8b8158bcb1bf194108 (127.15.168.129:42641), f7d247a1721943c49472305b5bbab7a6 (127.15.168.130:45579)
08:03:24.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.858894 16441 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e904ce805e75413abce8e621262820b8" candidate_uuid: "82c7f3b9c7774a5281eae28fb7e64216" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6741eeca17394f8b8158bcb1bf194108"
08:03:24.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.858945 16582 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e904ce805e75413abce8e621262820b8" candidate_uuid: "82c7f3b9c7774a5281eae28fb7e64216" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f7d247a1721943c49472305b5bbab7a6"
08:03:24.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.859097 16582 raft_consensus.cc:3058] T e904ce805e75413abce8e621262820b8 P f7d247a1721943c49472305b5bbab7a6 [term 0 FOLLOWER]: Advancing to term 1
08:03:24.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.859083 16441 raft_consensus.cc:3058] T e904ce805e75413abce8e621262820b8 P 6741eeca17394f8b8158bcb1bf194108 [term 0 FOLLOWER]: Advancing to term 1
08:03:24.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.861217 16441 raft_consensus.cc:2466] T e904ce805e75413abce8e621262820b8 P 6741eeca17394f8b8158bcb1bf194108 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82c7f3b9c7774a5281eae28fb7e64216 in term 1.
08:03:24.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.861218 16582 raft_consensus.cc:2466] T e904ce805e75413abce8e621262820b8 P f7d247a1721943c49472305b5bbab7a6 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82c7f3b9c7774a5281eae28fb7e64216 in term 1.
08:03:24.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.861649 16648 leader_election.cc:304] T e904ce805e75413abce8e621262820b8 P 82c7f3b9c7774a5281eae28fb7e64216 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6741eeca17394f8b8158bcb1bf194108, 82c7f3b9c7774a5281eae28fb7e64216; no voters:
08:03:24.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.861963 16804 raft_consensus.cc:2802] T e904ce805e75413abce8e621262820b8 P 82c7f3b9c7774a5281eae28fb7e64216 [term 1 FOLLOWER]: Leader election won for term 1
08:03:24.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.862461 16804 raft_consensus.cc:695] T e904ce805e75413abce8e621262820b8 P 82c7f3b9c7774a5281eae28fb7e64216 [term 1 LEADER]: Becoming Leader. State: Replica: 82c7f3b9c7774a5281eae28fb7e64216, State: Running, Role: LEADER
08:03:24.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.862989 16804 consensus_queue.cc:237] T e904ce805e75413abce8e621262820b8 P 82c7f3b9c7774a5281eae28fb7e64216 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6741eeca17394f8b8158bcb1bf194108" member_type: VOTER last_known_addr { host: "127.15.168.129" port: 42641 } } peers { permanent_uuid: "f7d247a1721943c49472305b5bbab7a6" member_type: VOTER last_known_addr { host: "127.15.168.130" port: 45579 } } peers { permanent_uuid: "82c7f3b9c7774a5281eae28fb7e64216" member_type: VOTER last_known_addr { host: "127.15.168.131" port: 40597 } }
08:03:24.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.868383 16076 catalog_manager.cc:5581] T e904ce805e75413abce8e621262820b8 P 82c7f3b9c7774a5281eae28fb7e64216 reported cstate change: term changed from 0 to 1, leader changed from <none> to 82c7f3b9c7774a5281eae28fb7e64216 (127.15.168.131). New cstate: current_term: 1 leader_uuid: "82c7f3b9c7774a5281eae28fb7e64216" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6741eeca17394f8b8158bcb1bf194108" member_type: VOTER last_known_addr { host: "127.15.168.129" port: 42641 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f7d247a1721943c49472305b5bbab7a6" member_type: VOTER last_known_addr { host: "127.15.168.130" port: 45579 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "82c7f3b9c7774a5281eae28fb7e64216" member_type: VOTER last_known_addr { host: "127.15.168.131" port: 40597 } health_report { overall_health: HEALTHY } } }
08:03:24.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.923729 16804 raft_consensus.cc:491] T fac3dfebf08c4f418ff9fa5397d50084 P 82c7f3b9c7774a5281eae28fb7e64216 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:03:24.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.923952 16804 raft_consensus.cc:513] T fac3dfebf08c4f418ff9fa5397d50084 P 82c7f3b9c7774a5281eae28fb7e64216 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82c7f3b9c7774a5281eae28fb7e64216" member_type: VOTER last_known_addr { host: "127.15.168.131" port: 40597 } } peers { permanent_uuid: "6741eeca17394f8b8158bcb1bf194108" member_type: VOTER last_known_addr { host: "127.15.168.129" port: 42641 } } peers { permanent_uuid: "f7d247a1721943c49472305b5bbab7a6" member_type: VOTER last_known_addr { host: "127.15.168.130" port: 45579 } }
08:03:24.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.924518 16804 leader_election.cc:290] T fac3dfebf08c4f418ff9fa5397d50084 P 82c7f3b9c7774a5281eae28fb7e64216 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6741eeca17394f8b8158bcb1bf194108 (127.15.168.129:42641), f7d247a1721943c49472305b5bbab7a6 (127.15.168.130:45579)
08:03:24.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.925112 16441 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fac3dfebf08c4f418ff9fa5397d50084" candidate_uuid: "82c7f3b9c7774a5281eae28fb7e64216" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6741eeca17394f8b8158bcb1bf194108" is_pre_election: true
08:03:24.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.925235 16582 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fac3dfebf08c4f418ff9fa5397d50084" candidate_uuid: "82c7f3b9c7774a5281eae28fb7e64216" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f7d247a1721943c49472305b5bbab7a6" is_pre_election: true
08:03:24.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.925328 16441 raft_consensus.cc:2466] T fac3dfebf08c4f418ff9fa5397d50084 P 6741eeca17394f8b8158bcb1bf194108 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82c7f3b9c7774a5281eae28fb7e64216 in term 0.
08:03:24.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.925530 16582 raft_consensus.cc:2466] T fac3dfebf08c4f418ff9fa5397d50084 P f7d247a1721943c49472305b5bbab7a6 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82c7f3b9c7774a5281eae28fb7e64216 in term 0.
08:03:24.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.925769 16648 leader_election.cc:304] T fac3dfebf08c4f418ff9fa5397d50084 P 82c7f3b9c7774a5281eae28fb7e64216 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6741eeca17394f8b8158bcb1bf194108, 82c7f3b9c7774a5281eae28fb7e64216; no voters:
08:03:24.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.926075 16804 raft_consensus.cc:2802] T fac3dfebf08c4f418ff9fa5397d50084 P 82c7f3b9c7774a5281eae28fb7e64216 [term 0 FOLLOWER]: Leader pre-election won for term 1
08:03:24.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.926169 16804 raft_consensus.cc:491] T fac3dfebf08c4f418ff9fa5397d50084 P 82c7f3b9c7774a5281eae28fb7e64216 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:03:24.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.926292 16804 raft_consensus.cc:3058] T fac3dfebf08c4f418ff9fa5397d50084 P 82c7f3b9c7774a5281eae28fb7e64216 [term 0 FOLLOWER]: Advancing to term 1
08:03:24.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.928290 16804 raft_consensus.cc:513] T fac3dfebf08c4f418ff9fa5397d50084 P 82c7f3b9c7774a5281eae28fb7e64216 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82c7f3b9c7774a5281eae28fb7e64216" member_type: VOTER last_known_addr { host: "127.15.168.131" port: 40597 } } peers { permanent_uuid: "6741eeca17394f8b8158bcb1bf194108" member_type: VOTER last_known_addr { host: "127.15.168.129" port: 42641 } } peers { permanent_uuid: "f7d247a1721943c49472305b5bbab7a6" member_type: VOTER last_known_addr { host: "127.15.168.130" port: 45579 } }
08:03:24.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.928881 16804 leader_election.cc:290] T fac3dfebf08c4f418ff9fa5397d50084 P 82c7f3b9c7774a5281eae28fb7e64216 [CANDIDATE]: Term 1 election: Requested vote from peers 6741eeca17394f8b8158bcb1bf194108 (127.15.168.129:42641), f7d247a1721943c49472305b5bbab7a6 (127.15.168.130:45579)
08:03:24.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.929430 16441 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fac3dfebf08c4f418ff9fa5397d50084" candidate_uuid: "82c7f3b9c7774a5281eae28fb7e64216" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6741eeca17394f8b8158bcb1bf194108"
08:03:24.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.929448 16582 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fac3dfebf08c4f418ff9fa5397d50084" candidate_uuid: "82c7f3b9c7774a5281eae28fb7e64216" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f7d247a1721943c49472305b5bbab7a6"
08:03:24.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.929623 16582 raft_consensus.cc:3058] T fac3dfebf08c4f418ff9fa5397d50084 P f7d247a1721943c49472305b5bbab7a6 [term 0 FOLLOWER]: Advancing to term 1
08:03:24.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.929644 16441 raft_consensus.cc:3058] T fac3dfebf08c4f418ff9fa5397d50084 P 6741eeca17394f8b8158bcb1bf194108 [term 0 FOLLOWER]: Advancing to term 1
08:03:24.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.931695 16582 raft_consensus.cc:2466] T fac3dfebf08c4f418ff9fa5397d50084 P f7d247a1721943c49472305b5bbab7a6 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82c7f3b9c7774a5281eae28fb7e64216 in term 1.
08:03:24.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.931788 16441 raft_consensus.cc:2466] T fac3dfebf08c4f418ff9fa5397d50084 P 6741eeca17394f8b8158bcb1bf194108 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82c7f3b9c7774a5281eae28fb7e64216 in term 1.
08:03:24.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.932137 16648 leader_election.cc:304] T fac3dfebf08c4f418ff9fa5397d50084 P 82c7f3b9c7774a5281eae28fb7e64216 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 82c7f3b9c7774a5281eae28fb7e64216, f7d247a1721943c49472305b5bbab7a6; no voters:
08:03:24.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.932425 16804 raft_consensus.cc:2802] T fac3dfebf08c4f418ff9fa5397d50084 P 82c7f3b9c7774a5281eae28fb7e64216 [term 1 FOLLOWER]: Leader election won for term 1
08:03:24.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.932598 16804 raft_consensus.cc:695] T fac3dfebf08c4f418ff9fa5397d50084 P 82c7f3b9c7774a5281eae28fb7e64216 [term 1 LEADER]: Becoming Leader. State: Replica: 82c7f3b9c7774a5281eae28fb7e64216, State: Running, Role: LEADER
08:03:24.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.932888 16804 consensus_queue.cc:237] T fac3dfebf08c4f418ff9fa5397d50084 P 82c7f3b9c7774a5281eae28fb7e64216 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82c7f3b9c7774a5281eae28fb7e64216" member_type: VOTER last_known_addr { host: "127.15.168.131" port: 40597 } } peers { permanent_uuid: "6741eeca17394f8b8158bcb1bf194108" member_type: VOTER last_known_addr { host: "127.15.168.129" port: 42641 } } peers { permanent_uuid: "f7d247a1721943c49472305b5bbab7a6" member_type: VOTER last_known_addr { host: "127.15.168.130" port: 45579 } }
08:03:24.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:24.935961 16076 catalog_manager.cc:5581] T fac3dfebf08c4f418ff9fa5397d50084 P 82c7f3b9c7774a5281eae28fb7e64216 reported cstate change: term changed from 0 to 1, leader changed from <none> to 82c7f3b9c7774a5281eae28fb7e64216 (127.15.168.131). New cstate: current_term: 1 leader_uuid: "82c7f3b9c7774a5281eae28fb7e64216" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82c7f3b9c7774a5281eae28fb7e64216" member_type: VOTER last_known_addr { host: "127.15.168.131" port: 40597 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6741eeca17394f8b8158bcb1bf194108" member_type: VOTER last_known_addr { host: "127.15.168.129" port: 42641 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f7d247a1721943c49472305b5bbab7a6" member_type: VOTER last_known_addr { host: "127.15.168.130" port: 45579 } health_report { overall_health: UNKNOWN } } }
08:03:25.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.361541 16804 consensus_queue.cc:1035] T fac3dfebf08c4f418ff9fa5397d50084 P 82c7f3b9c7774a5281eae28fb7e64216 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f7d247a1721943c49472305b5bbab7a6" member_type: VOTER last_known_addr { host: "127.15.168.130" port: 45579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:03:25.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.367851 16806 consensus_queue.cc:1035] T e904ce805e75413abce8e621262820b8 P 82c7f3b9c7774a5281eae28fb7e64216 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6741eeca17394f8b8158bcb1bf194108" member_type: VOTER last_known_addr { host: "127.15.168.129" port: 42641 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:03:25.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.371641 16806 consensus_queue.cc:1035] T fac3dfebf08c4f418ff9fa5397d50084 P 82c7f3b9c7774a5281eae28fb7e64216 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6741eeca17394f8b8158bcb1bf194108" member_type: VOTER last_known_addr { host: "127.15.168.129" port: 42641 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:03:25.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.382663 16804 consensus_queue.cc:1035] T e904ce805e75413abce8e621262820b8 P 82c7f3b9c7774a5281eae28fb7e64216 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f7d247a1721943c49472305b5bbab7a6" member_type: VOTER last_known_addr { host: "127.15.168.130" port: 45579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:03:25.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.553352 16075 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:50674:
08:03:25.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
08:03:25.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
08:03:25.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:25.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
08:03:25.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
08:03:25.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:03:25.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:25.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:03:25.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:03:25.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:25.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:25.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:25.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:25.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
08:03:25.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
08:03:25.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:03:25.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:25.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:03:25.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:03:25.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:25.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:25.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:25.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:25.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
08:03:25.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
08:03:25.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:03:25.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:25.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:03:25.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:03:25.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:25.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:25.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:25.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:25.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
08:03:25.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
08:03:25.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
08:03:25.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:25.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:03:25.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:03:25.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:25.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:25.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:25.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:25.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
08:03:25.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
08:03:25.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
08:03:25.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
08:03:25.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:25.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
08:03:25.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
08:03:25.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:25.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
08:03:25.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:25.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:25.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:25.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
08:03:25.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:25.554479 16075 catalog_manager.cc:6943] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
08:03:25.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.570662 16700 tablet_service.cc:1467] Processing CreateTablet for tablet eec0660dc1864bc9843e00f0f615364e (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=9b2524530e84418583c240c36b4d6d52]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
08:03:25.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.570663 16421 tablet_service.cc:1467] Processing CreateTablet for tablet eec0660dc1864bc9843e00f0f615364e (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=9b2524530e84418583c240c36b4d6d52]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
08:03:25.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.571795 16421 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet eec0660dc1864bc9843e00f0f615364e. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:25.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.571795 16700 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet eec0660dc1864bc9843e00f0f615364e. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:25.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.575201 16561 tablet_service.cc:1467] Processing CreateTablet for tablet eec0660dc1864bc9843e00f0f615364e (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=9b2524530e84418583c240c36b4d6d52]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
08:03:25.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.576279 16561 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet eec0660dc1864bc9843e00f0f615364e. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:25.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.581486 16839 tablet_bootstrap.cc:492] T eec0660dc1864bc9843e00f0f615364e P 6741eeca17394f8b8158bcb1bf194108: Bootstrap starting.
08:03:25.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.582319 16840 tablet_bootstrap.cc:492] T eec0660dc1864bc9843e00f0f615364e P 82c7f3b9c7774a5281eae28fb7e64216: Bootstrap starting.
08:03:25.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.583632 16839 tablet_bootstrap.cc:654] T eec0660dc1864bc9843e00f0f615364e P 6741eeca17394f8b8158bcb1bf194108: Neither blocks nor log segments found. Creating new log.
08:03:25.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.584784 16841 tablet_bootstrap.cc:492] T eec0660dc1864bc9843e00f0f615364e P f7d247a1721943c49472305b5bbab7a6: Bootstrap starting.
08:03:25.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.585050 16840 tablet_bootstrap.cc:654] T eec0660dc1864bc9843e00f0f615364e P 82c7f3b9c7774a5281eae28fb7e64216: Neither blocks nor log segments found. Creating new log.
08:03:25.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.586171 16839 tablet_bootstrap.cc:492] T eec0660dc1864bc9843e00f0f615364e P 6741eeca17394f8b8158bcb1bf194108: No bootstrap required, opened a new log
08:03:25.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.586318 16839 ts_tablet_manager.cc:1397] T eec0660dc1864bc9843e00f0f615364e P 6741eeca17394f8b8158bcb1bf194108: Time spent bootstrapping tablet: real 0.005s user 0.004s sys 0.000s
08:03:25.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.586566 16841 tablet_bootstrap.cc:654] T eec0660dc1864bc9843e00f0f615364e P f7d247a1721943c49472305b5bbab7a6: Neither blocks nor log segments found. Creating new log.
08:03:25.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.587091 16840 tablet_bootstrap.cc:492] T eec0660dc1864bc9843e00f0f615364e P 82c7f3b9c7774a5281eae28fb7e64216: No bootstrap required, opened a new log
08:03:25.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.587246 16839 raft_consensus.cc:357] T eec0660dc1864bc9843e00f0f615364e P 6741eeca17394f8b8158bcb1bf194108 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6741eeca17394f8b8158bcb1bf194108" member_type: VOTER last_known_addr { host: "127.15.168.129" port: 42641 } } peers { permanent_uuid: "f7d247a1721943c49472305b5bbab7a6" member_type: VOTER last_known_addr { host: "127.15.168.130" port: 45579 } } peers { permanent_uuid: "82c7f3b9c7774a5281eae28fb7e64216" member_type: VOTER last_known_addr { host: "127.15.168.131" port: 40597 } }
08:03:25.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.587497 16839 raft_consensus.cc:383] T eec0660dc1864bc9843e00f0f615364e P 6741eeca17394f8b8158bcb1bf194108 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:25.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.587600 16839 raft_consensus.cc:738] T eec0660dc1864bc9843e00f0f615364e P 6741eeca17394f8b8158bcb1bf194108 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6741eeca17394f8b8158bcb1bf194108, State: Initialized, Role: FOLLOWER
08:03:25.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.587937 16840 ts_tablet_manager.cc:1397] T eec0660dc1864bc9843e00f0f615364e P 82c7f3b9c7774a5281eae28fb7e64216: Time spent bootstrapping tablet: real 0.006s user 0.004s sys 0.000s
08:03:25.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.587842 16839 consensus_queue.cc:260] T eec0660dc1864bc9843e00f0f615364e P 6741eeca17394f8b8158bcb1bf194108 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6741eeca17394f8b8158bcb1bf194108" member_type: VOTER last_known_addr { host: "127.15.168.129" port: 42641 } } peers { permanent_uuid: "f7d247a1721943c49472305b5bbab7a6" member_type: VOTER last_known_addr { host: "127.15.168.130" port: 45579 } } peers { permanent_uuid: "82c7f3b9c7774a5281eae28fb7e64216" member_type: VOTER last_known_addr { host: "127.15.168.131" port: 40597 } }
08:03:25.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.588464 16839 ts_tablet_manager.cc:1428] T eec0660dc1864bc9843e00f0f615364e P 6741eeca17394f8b8158bcb1bf194108: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
08:03:25.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.588413 16815 tablet_replica.cc:440] T eec0660dc1864bc9843e00f0f615364e P 6741eeca17394f8b8158bcb1bf194108: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6741eeca17394f8b8158bcb1bf194108" member_type: VOTER last_known_addr { host: "127.15.168.129" port: 42641 } } peers { permanent_uuid: "f7d247a1721943c49472305b5bbab7a6" member_type: VOTER last_known_addr { host: "127.15.168.130" port: 45579 } } peers { permanent_uuid: "82c7f3b9c7774a5281eae28fb7e64216" member_type: VOTER last_known_addr { host: "127.15.168.131" port: 40597 } } }
08:03:25.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.588732 16815 tablet_replica.cc:443] T eec0660dc1864bc9843e00f0f615364e P 6741eeca17394f8b8158bcb1bf194108: This TxnStatusTablet replica's current role is: FOLLOWER
08:03:25.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.590965 16840 raft_consensus.cc:357] T eec0660dc1864bc9843e00f0f615364e P 82c7f3b9c7774a5281eae28fb7e64216 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6741eeca17394f8b8158bcb1bf194108" member_type: VOTER last_known_addr { host: "127.15.168.129" port: 42641 } } peers { permanent_uuid: "f7d247a1721943c49472305b5bbab7a6" member_type: VOTER last_known_addr { host: "127.15.168.130" port: 45579 } } peers { permanent_uuid: "82c7f3b9c7774a5281eae28fb7e64216" member_type: VOTER last_known_addr { host: "127.15.168.131" port: 40597 } }
08:03:25.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.591296 16840 raft_consensus.cc:383] T eec0660dc1864bc9843e00f0f615364e P 82c7f3b9c7774a5281eae28fb7e64216 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:25.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.591516 16840 raft_consensus.cc:738] T eec0660dc1864bc9843e00f0f615364e P 82c7f3b9c7774a5281eae28fb7e64216 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82c7f3b9c7774a5281eae28fb7e64216, State: Initialized, Role: FOLLOWER
08:03:25.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.591858 16840 consensus_queue.cc:260] T eec0660dc1864bc9843e00f0f615364e P 82c7f3b9c7774a5281eae28fb7e64216 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6741eeca17394f8b8158bcb1bf194108" member_type: VOTER last_known_addr { host: "127.15.168.129" port: 42641 } } peers { permanent_uuid: "f7d247a1721943c49472305b5bbab7a6" member_type: VOTER last_known_addr { host: "127.15.168.130" port: 45579 } } peers { permanent_uuid: "82c7f3b9c7774a5281eae28fb7e64216" member_type: VOTER last_known_addr { host: "127.15.168.131" port: 40597 } }
08:03:25.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.592412 16804 tablet_replica.cc:440] T eec0660dc1864bc9843e00f0f615364e P 82c7f3b9c7774a5281eae28fb7e64216: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6741eeca17394f8b8158bcb1bf194108" member_type: VOTER last_known_addr { host: "127.15.168.129" port: 42641 } } peers { permanent_uuid: "f7d247a1721943c49472305b5bbab7a6" member_type: VOTER last_known_addr { host: "127.15.168.130" port: 45579 } } peers { permanent_uuid: "82c7f3b9c7774a5281eae28fb7e64216" member_type: VOTER last_known_addr { host: "127.15.168.131" port: 40597 } } }
08:03:25.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.592624 16804 tablet_replica.cc:443] T eec0660dc1864bc9843e00f0f615364e P 82c7f3b9c7774a5281eae28fb7e64216: This TxnStatusTablet replica's current role is: FOLLOWER
08:03:25.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.595053 16840 ts_tablet_manager.cc:1428] T eec0660dc1864bc9843e00f0f615364e P 82c7f3b9c7774a5281eae28fb7e64216: Time spent starting tablet: real 0.005s user 0.002s sys 0.000s
08:03:25.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.597816 16841 tablet_bootstrap.cc:492] T eec0660dc1864bc9843e00f0f615364e P f7d247a1721943c49472305b5bbab7a6: No bootstrap required, opened a new log
08:03:25.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.597951 16841 ts_tablet_manager.cc:1397] T eec0660dc1864bc9843e00f0f615364e P f7d247a1721943c49472305b5bbab7a6: Time spent bootstrapping tablet: real 0.013s user 0.004s sys 0.005s
08:03:25.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.598582 16841 raft_consensus.cc:357] T eec0660dc1864bc9843e00f0f615364e P f7d247a1721943c49472305b5bbab7a6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6741eeca17394f8b8158bcb1bf194108" member_type: VOTER last_known_addr { host: "127.15.168.129" port: 42641 } } peers { permanent_uuid: "f7d247a1721943c49472305b5bbab7a6" member_type: VOTER last_known_addr { host: "127.15.168.130" port: 45579 } } peers { permanent_uuid: "82c7f3b9c7774a5281eae28fb7e64216" member_type: VOTER last_known_addr { host: "127.15.168.131" port: 40597 } }
08:03:25.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.598722 16841 raft_consensus.cc:383] T eec0660dc1864bc9843e00f0f615364e P f7d247a1721943c49472305b5bbab7a6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:25.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.598788 16841 raft_consensus.cc:738] T eec0660dc1864bc9843e00f0f615364e P f7d247a1721943c49472305b5bbab7a6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f7d247a1721943c49472305b5bbab7a6, State: Initialized, Role: FOLLOWER
08:03:25.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.598999 16841 consensus_queue.cc:260] T eec0660dc1864bc9843e00f0f615364e P f7d247a1721943c49472305b5bbab7a6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6741eeca17394f8b8158bcb1bf194108" member_type: VOTER last_known_addr { host: "127.15.168.129" port: 42641 } } peers { permanent_uuid: "f7d247a1721943c49472305b5bbab7a6" member_type: VOTER last_known_addr { host: "127.15.168.130" port: 45579 } } peers { permanent_uuid: "82c7f3b9c7774a5281eae28fb7e64216" member_type: VOTER last_known_addr { host: "127.15.168.131" port: 40597 } }
08:03:25.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.599452 16810 tablet_replica.cc:440] T eec0660dc1864bc9843e00f0f615364e P f7d247a1721943c49472305b5bbab7a6: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6741eeca17394f8b8158bcb1bf194108" member_type: VOTER last_known_addr { host: "127.15.168.129" port: 42641 } } peers { permanent_uuid: "f7d247a1721943c49472305b5bbab7a6" member_type: VOTER last_known_addr { host: "127.15.168.130" port: 45579 } } peers { permanent_uuid: "82c7f3b9c7774a5281eae28fb7e64216" member_type: VOTER last_known_addr { host: "127.15.168.131" port: 40597 } } }
08:03:25.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.599766 16810 tablet_replica.cc:443] T eec0660dc1864bc9843e00f0f615364e P f7d247a1721943c49472305b5bbab7a6: This TxnStatusTablet replica's current role is: FOLLOWER
08:03:25.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.600148 16841 ts_tablet_manager.cc:1428] T eec0660dc1864bc9843e00f0f615364e P f7d247a1721943c49472305b5bbab7a6: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
08:03:25.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.622202 16804 raft_consensus.cc:491] T eec0660dc1864bc9843e00f0f615364e P 82c7f3b9c7774a5281eae28fb7e64216 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:03:25.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.622414 16804 raft_consensus.cc:513] T eec0660dc1864bc9843e00f0f615364e P 82c7f3b9c7774a5281eae28fb7e64216 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6741eeca17394f8b8158bcb1bf194108" member_type: VOTER last_known_addr { host: "127.15.168.129" port: 42641 } } peers { permanent_uuid: "f7d247a1721943c49472305b5bbab7a6" member_type: VOTER last_known_addr { host: "127.15.168.130" port: 45579 } } peers { permanent_uuid: "82c7f3b9c7774a5281eae28fb7e64216" member_type: VOTER last_known_addr { host: "127.15.168.131" port: 40597 } }
08:03:25.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.623231 16804 leader_election.cc:290] T eec0660dc1864bc9843e00f0f615364e P 82c7f3b9c7774a5281eae28fb7e64216 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6741eeca17394f8b8158bcb1bf194108 (127.15.168.129:42641), f7d247a1721943c49472305b5bbab7a6 (127.15.168.130:45579)
08:03:25.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.623801 16440 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "eec0660dc1864bc9843e00f0f615364e" candidate_uuid: "82c7f3b9c7774a5281eae28fb7e64216" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6741eeca17394f8b8158bcb1bf194108" is_pre_election: true
08:03:25.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.623890 16582 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "eec0660dc1864bc9843e00f0f615364e" candidate_uuid: "82c7f3b9c7774a5281eae28fb7e64216" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f7d247a1721943c49472305b5bbab7a6" is_pre_election: true
08:03:25.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.624027 16440 raft_consensus.cc:2466] T eec0660dc1864bc9843e00f0f615364e P 6741eeca17394f8b8158bcb1bf194108 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82c7f3b9c7774a5281eae28fb7e64216 in term 0.
08:03:25.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.624133 16582 raft_consensus.cc:2466] T eec0660dc1864bc9843e00f0f615364e P f7d247a1721943c49472305b5bbab7a6 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82c7f3b9c7774a5281eae28fb7e64216 in term 0.
08:03:25.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.624445 16648 leader_election.cc:304] T eec0660dc1864bc9843e00f0f615364e P 82c7f3b9c7774a5281eae28fb7e64216 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6741eeca17394f8b8158bcb1bf194108, 82c7f3b9c7774a5281eae28fb7e64216; no voters:
08:03:25.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.624734 16804 raft_consensus.cc:2802] T eec0660dc1864bc9843e00f0f615364e P 82c7f3b9c7774a5281eae28fb7e64216 [term 0 FOLLOWER]: Leader pre-election won for term 1
08:03:25.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.624855 16804 raft_consensus.cc:491] T eec0660dc1864bc9843e00f0f615364e P 82c7f3b9c7774a5281eae28fb7e64216 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:03:25.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.624962 16804 raft_consensus.cc:3058] T eec0660dc1864bc9843e00f0f615364e P 82c7f3b9c7774a5281eae28fb7e64216 [term 0 FOLLOWER]: Advancing to term 1
08:03:25.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.626780 16804 raft_consensus.cc:513] T eec0660dc1864bc9843e00f0f615364e P 82c7f3b9c7774a5281eae28fb7e64216 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6741eeca17394f8b8158bcb1bf194108" member_type: VOTER last_known_addr { host: "127.15.168.129" port: 42641 } } peers { permanent_uuid: "f7d247a1721943c49472305b5bbab7a6" member_type: VOTER last_known_addr { host: "127.15.168.130" port: 45579 } } peers { permanent_uuid: "82c7f3b9c7774a5281eae28fb7e64216" member_type: VOTER last_known_addr { host: "127.15.168.131" port: 40597 } }
08:03:25.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.627413 16804 leader_election.cc:290] T eec0660dc1864bc9843e00f0f615364e P 82c7f3b9c7774a5281eae28fb7e64216 [CANDIDATE]: Term 1 election: Requested vote from peers 6741eeca17394f8b8158bcb1bf194108 (127.15.168.129:42641), f7d247a1721943c49472305b5bbab7a6 (127.15.168.130:45579)
08:03:25.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.627835 16440 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "eec0660dc1864bc9843e00f0f615364e" candidate_uuid: "82c7f3b9c7774a5281eae28fb7e64216" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6741eeca17394f8b8158bcb1bf194108"
08:03:25.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.628042 16440 raft_consensus.cc:3058] T eec0660dc1864bc9843e00f0f615364e P 6741eeca17394f8b8158bcb1bf194108 [term 0 FOLLOWER]: Advancing to term 1
08:03:25.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.628031 16582 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "eec0660dc1864bc9843e00f0f615364e" candidate_uuid: "82c7f3b9c7774a5281eae28fb7e64216" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f7d247a1721943c49472305b5bbab7a6"
08:03:25.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.628263 16582 raft_consensus.cc:3058] T eec0660dc1864bc9843e00f0f615364e P f7d247a1721943c49472305b5bbab7a6 [term 0 FOLLOWER]: Advancing to term 1
08:03:25.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.630375 16440 raft_consensus.cc:2466] T eec0660dc1864bc9843e00f0f615364e P 6741eeca17394f8b8158bcb1bf194108 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82c7f3b9c7774a5281eae28fb7e64216 in term 1.
08:03:25.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.630537 16582 raft_consensus.cc:2466] T eec0660dc1864bc9843e00f0f615364e P f7d247a1721943c49472305b5bbab7a6 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82c7f3b9c7774a5281eae28fb7e64216 in term 1.
08:03:25.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.630859 16648 leader_election.cc:304] T eec0660dc1864bc9843e00f0f615364e P 82c7f3b9c7774a5281eae28fb7e64216 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6741eeca17394f8b8158bcb1bf194108, 82c7f3b9c7774a5281eae28fb7e64216; no voters:
08:03:25.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.631227 16804 raft_consensus.cc:2802] T eec0660dc1864bc9843e00f0f615364e P 82c7f3b9c7774a5281eae28fb7e64216 [term 1 FOLLOWER]: Leader election won for term 1
08:03:25.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.631383 16804 raft_consensus.cc:695] T eec0660dc1864bc9843e00f0f615364e P 82c7f3b9c7774a5281eae28fb7e64216 [term 1 LEADER]: Becoming Leader. State: Replica: 82c7f3b9c7774a5281eae28fb7e64216, State: Running, Role: LEADER
08:03:25.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.631681 16804 consensus_queue.cc:237] T eec0660dc1864bc9843e00f0f615364e P 82c7f3b9c7774a5281eae28fb7e64216 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6741eeca17394f8b8158bcb1bf194108" member_type: VOTER last_known_addr { host: "127.15.168.129" port: 42641 } } peers { permanent_uuid: "f7d247a1721943c49472305b5bbab7a6" member_type: VOTER last_known_addr { host: "127.15.168.130" port: 45579 } } peers { permanent_uuid: "82c7f3b9c7774a5281eae28fb7e64216" member_type: VOTER last_known_addr { host: "127.15.168.131" port: 40597 } }
08:03:25.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.633335 16806 tablet_replica.cc:440] T eec0660dc1864bc9843e00f0f615364e P 82c7f3b9c7774a5281eae28fb7e64216: TxnStatusTablet state changed. Reason: New leader 82c7f3b9c7774a5281eae28fb7e64216. Latest consensus state: current_term: 1 leader_uuid: "82c7f3b9c7774a5281eae28fb7e64216" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6741eeca17394f8b8158bcb1bf194108" member_type: VOTER last_known_addr { host: "127.15.168.129" port: 42641 } } peers { permanent_uuid: "f7d247a1721943c49472305b5bbab7a6" member_type: VOTER last_known_addr { host: "127.15.168.130" port: 45579 } } peers { permanent_uuid: "82c7f3b9c7774a5281eae28fb7e64216" member_type: VOTER last_known_addr { host: "127.15.168.131" port: 40597 } } }
08:03:25.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.633569 16806 tablet_replica.cc:443] T eec0660dc1864bc9843e00f0f615364e P 82c7f3b9c7774a5281eae28fb7e64216: This TxnStatusTablet replica's current role is: LEADER
08:03:25.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.634407 16846 txn_status_manager.cc:874] Waiting until node catch up with all replicated operations in previous term...
08:03:25.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.634677 16846 txn_status_manager.cc:930] Loading transaction status metadata into memory...
08:03:25.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.635118 16075 catalog_manager.cc:5581] T eec0660dc1864bc9843e00f0f615364e P 82c7f3b9c7774a5281eae28fb7e64216 reported cstate change: term changed from 0 to 1, leader changed from <none> to 82c7f3b9c7774a5281eae28fb7e64216 (127.15.168.131). New cstate: current_term: 1 leader_uuid: "82c7f3b9c7774a5281eae28fb7e64216" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6741eeca17394f8b8158bcb1bf194108" member_type: VOTER last_known_addr { host: "127.15.168.129" port: 42641 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f7d247a1721943c49472305b5bbab7a6" member_type: VOTER last_known_addr { host: "127.15.168.130" port: 45579 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "82c7f3b9c7774a5281eae28fb7e64216" member_type: VOTER last_known_addr { host: "127.15.168.131" port: 40597 } health_report { overall_health: HEALTHY } } }
08:03:25.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.667248 16440 raft_consensus.cc:1273] T eec0660dc1864bc9843e00f0f615364e P 6741eeca17394f8b8158bcb1bf194108 [term 1 FOLLOWER]: Refusing update from remote peer 82c7f3b9c7774a5281eae28fb7e64216: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:03:25.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.667330 16582 raft_consensus.cc:1273] T eec0660dc1864bc9843e00f0f615364e P f7d247a1721943c49472305b5bbab7a6 [term 1 FOLLOWER]: Refusing update from remote peer 82c7f3b9c7774a5281eae28fb7e64216: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:03:25.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.667980 16804 consensus_queue.cc:1035] T eec0660dc1864bc9843e00f0f615364e P 82c7f3b9c7774a5281eae28fb7e64216 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f7d247a1721943c49472305b5bbab7a6" member_type: VOTER last_known_addr { host: "127.15.168.130" port: 45579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:03:25.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.668319 16806 consensus_queue.cc:1035] T eec0660dc1864bc9843e00f0f615364e P 82c7f3b9c7774a5281eae28fb7e64216 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6741eeca17394f8b8158bcb1bf194108" member_type: VOTER last_known_addr { host: "127.15.168.129" port: 42641 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:03:25.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.676400 16810 tablet_replica.cc:440] T eec0660dc1864bc9843e00f0f615364e P f7d247a1721943c49472305b5bbab7a6: TxnStatusTablet state changed. Reason: New leader 82c7f3b9c7774a5281eae28fb7e64216. Latest consensus state: current_term: 1 leader_uuid: "82c7f3b9c7774a5281eae28fb7e64216" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6741eeca17394f8b8158bcb1bf194108" member_type: VOTER last_known_addr { host: "127.15.168.129" port: 42641 } } peers { permanent_uuid: "f7d247a1721943c49472305b5bbab7a6" member_type: VOTER last_known_addr { host: "127.15.168.130" port: 45579 } } peers { permanent_uuid: "82c7f3b9c7774a5281eae28fb7e64216" member_type: VOTER last_known_addr { host: "127.15.168.131" port: 40597 } } }
08:03:25.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.676729 16810 tablet_replica.cc:443] T eec0660dc1864bc9843e00f0f615364e P f7d247a1721943c49472305b5bbab7a6: This TxnStatusTablet replica's current role is: FOLLOWER
08:03:25.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.680398 16812 mvcc.cc:204] Tried to move back new op lower bound from 7144538954405437440 to 7144538954269798400. Current Snapshot: MvccSnapshot[applied={T|T < 7144538954405437440}]
08:03:25.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.681706 16815 tablet_replica.cc:440] T eec0660dc1864bc9843e00f0f615364e P 6741eeca17394f8b8158bcb1bf194108: TxnStatusTablet state changed. Reason: New leader 82c7f3b9c7774a5281eae28fb7e64216. Latest consensus state: current_term: 1 leader_uuid: "82c7f3b9c7774a5281eae28fb7e64216" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6741eeca17394f8b8158bcb1bf194108" member_type: VOTER last_known_addr { host: "127.15.168.129" port: 42641 } } peers { permanent_uuid: "f7d247a1721943c49472305b5bbab7a6" member_type: VOTER last_known_addr { host: "127.15.168.130" port: 45579 } } peers { permanent_uuid: "82c7f3b9c7774a5281eae28fb7e64216" member_type: VOTER last_known_addr { host: "127.15.168.131" port: 40597 } } }
08:03:25.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.681969 16815 tablet_replica.cc:443] T eec0660dc1864bc9843e00f0f615364e P 6741eeca17394f8b8158bcb1bf194108: This TxnStatusTablet replica's current role is: FOLLOWER
08:03:25.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.688428 16817 mvcc.cc:204] Tried to move back new op lower bound from 7144538954405437440 to 7144538954269798400. Current Snapshot: MvccSnapshot[applied={T|T < 7144538954405437440}]
08:03:25.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.690326 16810 tablet_replica.cc:440] T eec0660dc1864bc9843e00f0f615364e P f7d247a1721943c49472305b5bbab7a6: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "82c7f3b9c7774a5281eae28fb7e64216" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6741eeca17394f8b8158bcb1bf194108" member_type: VOTER last_known_addr { host: "127.15.168.129" port: 42641 } } peers { permanent_uuid: "f7d247a1721943c49472305b5bbab7a6" member_type: VOTER last_known_addr { host: "127.15.168.130" port: 45579 } } peers { permanent_uuid: "82c7f3b9c7774a5281eae28fb7e64216" member_type: VOTER last_known_addr { host: "127.15.168.131" port: 40597 } } }
08:03:25.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.690591 16810 tablet_replica.cc:443] T eec0660dc1864bc9843e00f0f615364e P f7d247a1721943c49472305b5bbab7a6: This TxnStatusTablet replica's current role is: FOLLOWER
08:03:25.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.693493 16815 tablet_replica.cc:440] T eec0660dc1864bc9843e00f0f615364e P 6741eeca17394f8b8158bcb1bf194108: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "82c7f3b9c7774a5281eae28fb7e64216" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6741eeca17394f8b8158bcb1bf194108" member_type: VOTER last_known_addr { host: "127.15.168.129" port: 42641 } } peers { permanent_uuid: "f7d247a1721943c49472305b5bbab7a6" member_type: VOTER last_known_addr { host: "127.15.168.130" port: 45579 } } peers { permanent_uuid: "82c7f3b9c7774a5281eae28fb7e64216" member_type: VOTER last_known_addr { host: "127.15.168.131" port: 40597 } } }
08:03:25.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.693696 16815 tablet_replica.cc:443] T eec0660dc1864bc9843e00f0f615364e P 6741eeca17394f8b8158bcb1bf194108: This TxnStatusTablet replica's current role is: FOLLOWER
08:03:25.697 [DEBUG - Time-limited test] (KuduTransaction.java:702) starting keepalive heartbeating with period 500 ms (txn ID 0)
08:03:25.699 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table c38065584d484ecb87535dca2fe900a1
08:03:25.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.704244 16804 tablet_replica.cc:440] T eec0660dc1864bc9843e00f0f615364e P 82c7f3b9c7774a5281eae28fb7e64216: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "82c7f3b9c7774a5281eae28fb7e64216" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6741eeca17394f8b8158bcb1bf194108" member_type: VOTER last_known_addr { host: "127.15.168.129" port: 42641 } } peers { permanent_uuid: "f7d247a1721943c49472305b5bbab7a6" member_type: VOTER last_known_addr { host: "127.15.168.130" port: 45579 } } peers { permanent_uuid: "82c7f3b9c7774a5281eae28fb7e64216" member_type: VOTER last_known_addr { host: "127.15.168.131" port: 40597 } } }
08:03:25.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.704572 16804 tablet_replica.cc:443] T eec0660dc1864bc9843e00f0f615364e P 82c7f3b9c7774a5281eae28fb7e64216: This TxnStatusTablet replica's current role is: LEADER
08:03:25.705 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet fac3dfebf08c4f418ff9fa5397d50084 for table 'txn_manager_fallback' with partition [0x00000000, 0x00000001)
08:03:25.706 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet e904ce805e75413abce8e621262820b8 for table 'txn_manager_fallback' with partition [0x00000001, 0x00000002)
08:03:25.706 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x00000000, upperBoundPartitionKey=0x00000001, ttl=299999, tablet=fac3dfebf08c4f418ff9fa5397d50084@[6741eeca17394f8b8158bcb1bf194108(127.15.168.129:42641),82c7f3b9c7774a5281eae28fb7e64216(127.15.168.131:40597)[L],f7d247a1721943c49472305b5bbab7a6(127.15.168.130:45579)]}, Tablet{lowerBoundPartitionKey=0x00000001, upperBoundPartitionKey=0x00000002, ttl=299999, tablet=e904ce805e75413abce8e621262820b8@[6741eeca17394f8b8158bcb1bf194108(127.15.168.129:42641),82c7f3b9c7774a5281eae28fb7e64216(127.15.168.131:40597)[L],f7d247a1721943c49472305b5bbab7a6(127.15.168.130:45579)]}, NonCoveredRange{lowerBoundPartitionKey=0x00000002, upperBoundPartitionKey=0x, ttl=299999}]
08:03:25.706 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=5), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.15.168.190:33711, [4ms] Sub RPC GetTableLocations: received response from server master-127.15.168.190:33711: OK, deferred=null) after lookup
08:03:25.707 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 82c7f3b9c7774a5281eae28fb7e64216(127.15.168.131:40597)] connecting to peer
08:03:25.709 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 82c7f3b9c7774a5281eae28fb7e64216(127.15.168.131:40597)] Successfully connected to peer
08:03:25.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.719723 16806 tablet_replica.cc:440] T eec0660dc1864bc9843e00f0f615364e P 82c7f3b9c7774a5281eae28fb7e64216: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "82c7f3b9c7774a5281eae28fb7e64216" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6741eeca17394f8b8158bcb1bf194108" member_type: VOTER last_known_addr { host: "127.15.168.129" port: 42641 } } peers { permanent_uuid: "f7d247a1721943c49472305b5bbab7a6" member_type: VOTER last_known_addr { host: "127.15.168.130" port: 45579 } } peers { permanent_uuid: "82c7f3b9c7774a5281eae28fb7e64216" member_type: VOTER last_known_addr { host: "127.15.168.131" port: 40597 } } }
08:03:25.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.720034 16806 tablet_replica.cc:443] T eec0660dc1864bc9843e00f0f615364e P 82c7f3b9c7774a5281eae28fb7e64216: This TxnStatusTablet replica's current role is: LEADER
08:03:25.727 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x16c44b5f, L:/127.0.0.1:57490 - R:/127.15.168.131:40597] using TOKEN/null
08:03:25.766 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.15.168.190:33711
08:03:25.767 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
08:03:25.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.767814 16034 external_mini_cluster.cc:1661] Attempting to check leaks for /tmp/dist-test-taskbKfKhV/build/asan/bin/kudu pid 16045
08:03:25.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.769411 16116 generic_service.cc:196] Checking for leaks (request via RPC)
08:03:25.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:25.899892 16034 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskbKfKhV/build/asan/bin/kudu with pid 16045
08:03:25.923 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.15.168.190:33711(127.15.168.190:33711)] handling channelInactive
08:03:25.923 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.168.190:33711(127.15.168.190:33711)] cleaning up while in state READY due to: connection closed
08:03:25.923 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:03:25.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:25.981434 16770 heartbeater.cc:646] Failed to heartbeat to 127.15.168.190:33711 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.15.168.190:33711: connect: Connection refused (error 111)
08:03:26.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.200448 16863 raft_consensus.cc:491] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 2a06e94e44f24a35b5da6da522938c92)
08:03:26.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.200510 16864 raft_consensus.cc:491] T 00000000000000000000000000000000 P e84d2a2c5f154a8f8da59be47f9d54db [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 2a06e94e44f24a35b5da6da522938c92)
08:03:26.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.200663 16863 raft_consensus.cc:513] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2a06e94e44f24a35b5da6da522938c92" member_type: VOTER last_known_addr { host: "127.15.168.190" port: 33711 } } peers { permanent_uuid: "e84d2a2c5f154a8f8da59be47f9d54db" member_type: VOTER last_known_addr { host: "127.15.168.189" port: 43117 } } peers { permanent_uuid: "5bee3c957fe9436399c523d7516cc493" member_type: VOTER last_known_addr { host: "127.15.168.188" port: 38185 } }
08:03:26.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.200724 16864 raft_consensus.cc:513] T 00000000000000000000000000000000 P e84d2a2c5f154a8f8da59be47f9d54db [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2a06e94e44f24a35b5da6da522938c92" member_type: VOTER last_known_addr { host: "127.15.168.190" port: 33711 } } peers { permanent_uuid: "e84d2a2c5f154a8f8da59be47f9d54db" member_type: VOTER last_known_addr { host: "127.15.168.189" port: 43117 } } peers { permanent_uuid: "5bee3c957fe9436399c523d7516cc493" member_type: VOTER last_known_addr { host: "127.15.168.188" port: 38185 } }
08:03:26.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.201861 16863 leader_election.cc:290] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 2a06e94e44f24a35b5da6da522938c92 (127.15.168.190:33711), e84d2a2c5f154a8f8da59be47f9d54db (127.15.168.189:43117)
08:03:26.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.202041 16864 leader_election.cc:290] T 00000000000000000000000000000000 P e84d2a2c5f154a8f8da59be47f9d54db [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 2a06e94e44f24a35b5da6da522938c92 (127.15.168.190:33711), 5bee3c957fe9436399c523d7516cc493 (127.15.168.188:38185)
08:03:26.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.202531 16166 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5bee3c957fe9436399c523d7516cc493" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "e84d2a2c5f154a8f8da59be47f9d54db" is_pre_election: true
08:03:26.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.202785 16166 raft_consensus.cc:2466] T 00000000000000000000000000000000 P e84d2a2c5f154a8f8da59be47f9d54db [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5bee3c957fe9436399c523d7516cc493 in term 1.
08:03:26.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.202796 16247 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e84d2a2c5f154a8f8da59be47f9d54db" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "5bee3c957fe9436399c523d7516cc493" is_pre_election: true
08:03:26.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.203037 16247 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e84d2a2c5f154a8f8da59be47f9d54db in term 1.
08:03:26.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:26.203218 16223 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.15.168.190:33711: connect: Connection refused (error 111)
08:03:26.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.203470 16225 leader_election.cc:304] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 5bee3c957fe9436399c523d7516cc493, e84d2a2c5f154a8f8da59be47f9d54db; no voters:
08:03:26.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.203580 16141 leader_election.cc:304] T 00000000000000000000000000000000 P e84d2a2c5f154a8f8da59be47f9d54db [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 5bee3c957fe9436399c523d7516cc493, e84d2a2c5f154a8f8da59be47f9d54db; no voters:
08:03:26.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.203896 16863 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493 [term 1 FOLLOWER]: Leader pre-election won for term 2
08:03:26.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.204020 16863 raft_consensus.cc:491] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 2a06e94e44f24a35b5da6da522938c92)
08:03:26.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.204142 16863 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493 [term 1 FOLLOWER]: Advancing to term 2
08:03:26.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.204157 16864 raft_consensus.cc:2802] T 00000000000000000000000000000000 P e84d2a2c5f154a8f8da59be47f9d54db [term 1 FOLLOWER]: Leader pre-election won for term 2
08:03:26.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.204274 16864 raft_consensus.cc:491] T 00000000000000000000000000000000 P e84d2a2c5f154a8f8da59be47f9d54db [term 1 FOLLOWER]: Starting leader election (detected failure of leader 2a06e94e44f24a35b5da6da522938c92)
08:03:26.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.204378 16864 raft_consensus.cc:3058] T 00000000000000000000000000000000 P e84d2a2c5f154a8f8da59be47f9d54db [term 1 FOLLOWER]: Advancing to term 2
08:03:26.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.207304 16863 raft_consensus.cc:513] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2a06e94e44f24a35b5da6da522938c92" member_type: VOTER last_known_addr { host: "127.15.168.190" port: 33711 } } peers { permanent_uuid: "e84d2a2c5f154a8f8da59be47f9d54db" member_type: VOTER last_known_addr { host: "127.15.168.189" port: 43117 } } peers { permanent_uuid: "5bee3c957fe9436399c523d7516cc493" member_type: VOTER last_known_addr { host: "127.15.168.188" port: 38185 } }
08:03:26.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.207537 16864 raft_consensus.cc:513] T 00000000000000000000000000000000 P e84d2a2c5f154a8f8da59be47f9d54db [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2a06e94e44f24a35b5da6da522938c92" member_type: VOTER last_known_addr { host: "127.15.168.190" port: 33711 } } peers { permanent_uuid: "e84d2a2c5f154a8f8da59be47f9d54db" member_type: VOTER last_known_addr { host: "127.15.168.189" port: 43117 } } peers { permanent_uuid: "5bee3c957fe9436399c523d7516cc493" member_type: VOTER last_known_addr { host: "127.15.168.188" port: 38185 } }
08:03:26.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.208438 16864 leader_election.cc:290] T 00000000000000000000000000000000 P e84d2a2c5f154a8f8da59be47f9d54db [CANDIDATE]: Term 2 election: Requested vote from peers 2a06e94e44f24a35b5da6da522938c92 (127.15.168.190:33711), 5bee3c957fe9436399c523d7516cc493 (127.15.168.188:38185)
08:03:26.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.208765 16166 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5bee3c957fe9436399c523d7516cc493" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "e84d2a2c5f154a8f8da59be47f9d54db"
08:03:26.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.208843 16247 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e84d2a2c5f154a8f8da59be47f9d54db" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "5bee3c957fe9436399c523d7516cc493"
08:03:26.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.209039 16166 raft_consensus.cc:2391] T 00000000000000000000000000000000 P e84d2a2c5f154a8f8da59be47f9d54db [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 5bee3c957fe9436399c523d7516cc493 in current term 2: Already voted for candidate e84d2a2c5f154a8f8da59be47f9d54db in this term.
08:03:26.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.209143 16247 raft_consensus.cc:2391] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate e84d2a2c5f154a8f8da59be47f9d54db in current term 2: Already voted for candidate 5bee3c957fe9436399c523d7516cc493 in this term.
08:03:26.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.209796 16863 leader_election.cc:290] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493 [CANDIDATE]: Term 2 election: Requested vote from peers 2a06e94e44f24a35b5da6da522938c92 (127.15.168.190:33711), e84d2a2c5f154a8f8da59be47f9d54db (127.15.168.189:43117)
08:03:26.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:26.209879 16223 leader_election.cc:336] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 2a06e94e44f24a35b5da6da522938c92 (127.15.168.190:33711): Network error: Client connection negotiation failed: client connection to 127.15.168.190:33711: connect: Connection refused (error 111)
08:03:26.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:26.210247 16142 leader_election.cc:336] T 00000000000000000000000000000000 P e84d2a2c5f154a8f8da59be47f9d54db [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 2a06e94e44f24a35b5da6da522938c92 (127.15.168.190:33711): Network error: Client connection negotiation failed: client connection to 127.15.168.190:33711: connect: Connection refused (error 111)
08:03:26.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:26.211081 16223 leader_election.cc:336] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 2a06e94e44f24a35b5da6da522938c92 (127.15.168.190:33711): Network error: Client connection negotiation failed: client connection to 127.15.168.190:33711: connect: Connection refused (error 111)
08:03:26.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.211232 16223 leader_election.cc:304] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5bee3c957fe9436399c523d7516cc493; no voters: 2a06e94e44f24a35b5da6da522938c92, e84d2a2c5f154a8f8da59be47f9d54db
08:03:26.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:26.211409 16142 leader_election.cc:336] T 00000000000000000000000000000000 P e84d2a2c5f154a8f8da59be47f9d54db [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 2a06e94e44f24a35b5da6da522938c92 (127.15.168.190:33711): Network error: Client connection negotiation failed: client connection to 127.15.168.190:33711: connect: Connection refused (error 111)
08:03:26.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.211501 16863 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
08:03:26.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.211549 16142 leader_election.cc:304] T 00000000000000000000000000000000 P e84d2a2c5f154a8f8da59be47f9d54db [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e84d2a2c5f154a8f8da59be47f9d54db; no voters: 2a06e94e44f24a35b5da6da522938c92, 5bee3c957fe9436399c523d7516cc493
08:03:26.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.211895 16864 raft_consensus.cc:2747] T 00000000000000000000000000000000 P e84d2a2c5f154a8f8da59be47f9d54db [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
08:03:26.220 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.15.168.190:33711(127.15.168.190:33711)] connecting to peer
08:03:26.222 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711
08:03:26.222 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.168.190:33711(127.15.168.190:33711)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711
08:03:26.222 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location master-127.15.168.190:33711(127.15.168.190:33711) for tablet Kudu Master: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711
08:03:26.237 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.15.168.190:33711(127.15.168.190:33711)] connecting to peer
08:03:26.237 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711
08:03:26.238 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.168.190:33711(127.15.168.190:33711)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711
08:03:26.238 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.168.190:33711: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711
08:03:26.238 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.168.188:38185,127.15.168.190:33711,127.15.168.189:43117; will retry
08:03:26.256 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.15.168.190:33711(127.15.168.190:33711)] connecting to peer
08:03:26.257 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711
08:03:26.258 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.168.190:33711(127.15.168.190:33711)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711
08:03:26.258 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.168.190:33711: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711
08:03:26.258 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.168.188:38185,127.15.168.190:33711,127.15.168.189:43117; will retry
08:03:26.277 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.15.168.190:33711(127.15.168.190:33711)] connecting to peer
08:03:26.278 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711
08:03:26.279 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.168.190:33711(127.15.168.190:33711)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711
08:03:26.279 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.168.190:33711: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711
08:03:26.279 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.168.188:38185,127.15.168.190:33711,127.15.168.189:43117; will retry
08:03:26.298 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.15.168.190:33711(127.15.168.190:33711)] connecting to peer
08:03:26.299 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711
08:03:26.299 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.168.190:33711(127.15.168.190:33711)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711
08:03:26.299 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.168.190:33711: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711
08:03:26.300 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.168.188:38185,127.15.168.190:33711,127.15.168.189:43117; will retry
08:03:26.317 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.15.168.190:33711(127.15.168.190:33711)] connecting to peer
08:03:26.317 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711
08:03:26.318 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.168.190:33711(127.15.168.190:33711)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711
08:03:26.318 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.168.190:33711: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711
08:03:26.319 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.168.188:38185,127.15.168.190:33711,127.15.168.189:43117; will retry
08:03:26.337 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.15.168.190:33711(127.15.168.190:33711)] connecting to peer
08:03:26.337 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711
08:03:26.338 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.168.190:33711(127.15.168.190:33711)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711
08:03:26.338 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.168.190:33711: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711
08:03:26.338 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.168.188:38185,127.15.168.190:33711,127.15.168.189:43117; will retry
08:03:26.437 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.15.168.190:33711(127.15.168.190:33711)] connecting to peer
08:03:26.438 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711
08:03:26.438 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.168.190:33711(127.15.168.190:33711)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711
08:03:26.438 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.168.190:33711: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711
08:03:26.440 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.168.188:38185,127.15.168.190:33711,127.15.168.189:43117; will retry
08:03:26.440 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=223), Traces: [0ms] sending RPC to server master-127.15.168.190:33711, [5ms] received response from server master-127.15.168.190:33711: Network error: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [5ms] delaying RPC due to: Network error: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.188:38185, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.190:33711, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.189:43117, [21ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.190:33711: Network error: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [21ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.188:38185: OK, [21ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.189:43117: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.15.168.188:38185,127.15.168.190:33711,127.15.168.189:43117) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.188:38185, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.190:33711, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.189:43117, [41ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.190:33711: Network error: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [41ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.188:38185: OK, [41ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.189:43117: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.15.168.188:38185,127.15.168.190:33711,127.15.168.189:43117) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.188:38185, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.190:33711, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.189:43117, [62ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.190:33711: Network error: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [62ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.188:38185: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.189:43117: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.15.168.188:38185,127.15.168.190:33711,127.15.168.189:43117) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.188:38185, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.190:33711, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.189:43117, [82ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.190:33711: Network error: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [82ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.188:38185: OK, [83ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.189:43117: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.15.168.188:38185,127.15.168.190:33711,127.15.168.189:43117) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.188:38185, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.190:33711, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.189:43117, [101ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.190:33711: Network error: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [101ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.188:38185: OK, [101ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.189:43117: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.15.168.188:38185,127.15.168.190:33711,127.15.168.189:43117) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.188:38185, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.190:33711, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.189:43117, [121ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.190:33711: Network error: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [121ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.188:38185: OK, [121ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.189:43117: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.15.168.188:38185,127.15.168.190:33711,127.15.168.189:43117) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.188:38185, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.190:33711, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.189:43117, [221ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.190:33711: Network error: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [222ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.188:38185: OK, [222ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.189:43117: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.15.168.188:38185,127.15.168.190:33711,127.15.168.189:43117) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, deferred=Deferred@708617836(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$117/349169453@6c85c31c -> (continuation of Deferred@628642946 after retry RPC after error@1026345056) -> (continuation of Deferred@1928070767 after retry RPC after error@808691213) -> (continuation of Deferred@1460152525 after retry RPC after error@205292932) -> (continuation of Deferred@510731538 after retry RPC after error@861928223) -> (continuation of Deferred@621274105 after retry RPC after error@1308574328) -> (continuation of Deferred@1521592160 after retry RPC after error@377104426), errback=org.apache.kudu.client.KuduTransaction$$Lambda$118/1749245344@cf81937 -> (continuation of Deferred@628642946 after retry RPC after error@1026345056) -> (continuation of Deferred@1928070767 after retry RPC after error@808691213) -> (continuation of Deferred@1460152525 after retry RPC after error@205292932) -> (continuation of Deferred@510731538 after retry RPC after error@861928223) -> (continuation of Deferred@621274105 after retry RPC after error@1308574328) -> (continuation of Deferred@1521592160 after retry RPC after error@377104426)))
08:03:26.441 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=223), Traces: [0ms] sending RPC to server master-127.15.168.190:33711, [5ms] received response from server master-127.15.168.190:33711: Network error: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [5ms] delaying RPC due to: Network error: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.188:38185, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.190:33711, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.189:43117, [21ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.190:33711: Network error: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [21ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.188:38185: OK, [21ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.189:43117: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.15.168.188:38185,127.15.168.190:33711,127.15.168.189:43117) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.188:38185, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.190:33711, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.189:43117, [41ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.190:33711: Network error: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [41ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.188:38185: OK, [41ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.189:43117: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.15.168.188:38185,127.15.168.190:33711,127.15.168.189:43117) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.188:38185, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.190:33711, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.189:43117, [62ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.190:33711: Network error: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [62ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.188:38185: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.189:43117: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.15.168.188:38185,127.15.168.190:33711,127.15.168.189:43117) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.188:38185, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.190:33711, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.189:43117, [82ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.190:33711: Network error: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [82ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.188:38185: OK, [83ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.189:43117: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.15.168.188:38185,127.15.168.190:33711,127.15.168.189:43117) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.188:38185, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.190:33711, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.189:43117, [101ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.190:33711: Network error: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [101ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.188:38185: OK, [101ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.189:43117: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.15.168.188:38185,127.15.168.190:33711,127.15.168.189:43117) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.188:38185, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.190:33711, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.189:43117, [121ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.190:33711: Network error: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [121ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.188:38185: OK, [121ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.189:43117: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.15.168.188:38185,127.15.168.190:33711,127.15.168.189:43117) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.188:38185, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.190:33711, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.189:43117, [221ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.190:33711: Network error: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [222ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.188:38185: OK, [222ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.189:43117: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.15.168.188:38185,127.15.168.190:33711,127.15.168.189:43117) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, deferred=Deferred@708617836(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$117/349169453@6c85c31c -> (continuation of Deferred@628642946 after retry RPC after error@1026345056) -> (continuation of Deferred@1928070767 after retry RPC after error@808691213) -> (continuation of Deferred@1460152525 after retry RPC after error@205292932) -> (continuation of Deferred@510731538 after retry RPC after error@861928223) -> (continuation of Deferred@621274105 after retry RPC after error@1308574328) -> (continuation of Deferred@1521592160 after retry RPC after error@377104426), errback=org.apache.kudu.client.KuduTransaction$$Lambda$118/1749245344@cf81937 -> (continuation of Deferred@628642946 after retry RPC after error@1026345056) -> (continuation of Deferred@1928070767 after retry RPC after error@808691213) -> (continuation of Deferred@1460152525 after retry RPC after error@205292932) -> (continuation of Deferred@510731538 after retry RPC after error@861928223) -> (continuation of Deferred@621274105 after retry RPC after error@1308574328) -> (continuation of Deferred@1521592160 after retry RPC after error@377104426)))
08:03:26.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.15.168.190:33711(127.15.168.190:33711)] connecting to peer
08:03:26.458 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711
08:03:26.458 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.168.190:33711(127.15.168.190:33711)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711
08:03:26.459 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.168.190:33711: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711
08:03:26.459 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.168.188:38185,127.15.168.190:33711,127.15.168.189:43117; will retry
08:03:26.477 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.15.168.190:33711(127.15.168.190:33711)] connecting to peer
08:03:26.479 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711
08:03:26.479 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.168.190:33711(127.15.168.190:33711)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711
08:03:26.480 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.168.190:33711: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711
08:03:26.480 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.168.188:38185,127.15.168.190:33711,127.15.168.189:43117; will retry
08:03:26.497 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.15.168.190:33711(127.15.168.190:33711)] connecting to peer
08:03:26.499 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711
08:03:26.499 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.168.190:33711(127.15.168.190:33711)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711
08:03:26.499 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.168.190:33711: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711
08:03:26.500 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.168.188:38185,127.15.168.190:33711,127.15.168.189:43117; will retry
08:03:26.517 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.15.168.190:33711(127.15.168.190:33711)] connecting to peer
08:03:26.518 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711
08:03:26.518 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.168.190:33711(127.15.168.190:33711)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711
08:03:26.518 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.168.190:33711: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711
08:03:26.519 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.168.188:38185,127.15.168.190:33711,127.15.168.189:43117; will retry
08:03:26.537 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.15.168.190:33711(127.15.168.190:33711)] connecting to peer
08:03:26.538 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711
08:03:26.538 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.168.190:33711(127.15.168.190:33711)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711
08:03:26.538 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.168.190:33711: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711
08:03:26.539 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.168.188:38185,127.15.168.190:33711,127.15.168.189:43117; will retry
08:03:26.557 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.15.168.190:33711(127.15.168.190:33711)] connecting to peer
08:03:26.558 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711
08:03:26.558 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.168.190:33711(127.15.168.190:33711)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711
08:03:26.558 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.168.190:33711: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711
08:03:26.559 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.168.188:38185,127.15.168.190:33711,127.15.168.189:43117; will retry
08:03:26.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.565649 16863 raft_consensus.cc:491] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:03:26.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.565919 16863 raft_consensus.cc:513] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2a06e94e44f24a35b5da6da522938c92" member_type: VOTER last_known_addr { host: "127.15.168.190" port: 33711 } } peers { permanent_uuid: "e84d2a2c5f154a8f8da59be47f9d54db" member_type: VOTER last_known_addr { host: "127.15.168.189" port: 43117 } } peers { permanent_uuid: "5bee3c957fe9436399c523d7516cc493" member_type: VOTER last_known_addr { host: "127.15.168.188" port: 38185 } }
08:03:26.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.566608 16863 leader_election.cc:290] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 2a06e94e44f24a35b5da6da522938c92 (127.15.168.190:33711), e84d2a2c5f154a8f8da59be47f9d54db (127.15.168.189:43117)
08:03:26.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.567329 16166 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5bee3c957fe9436399c523d7516cc493" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "e84d2a2c5f154a8f8da59be47f9d54db" is_pre_election: true
08:03:26.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.567631 16166 raft_consensus.cc:2466] T 00000000000000000000000000000000 P e84d2a2c5f154a8f8da59be47f9d54db [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5bee3c957fe9436399c523d7516cc493 in term 2.
08:03:26.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:26.568089 16223 leader_election.cc:336] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 2a06e94e44f24a35b5da6da522938c92 (127.15.168.190:33711): Network error: Client connection negotiation failed: client connection to 127.15.168.190:33711: connect: Connection refused (error 111)
08:03:26.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.568307 16225 leader_election.cc:304] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5bee3c957fe9436399c523d7516cc493, e84d2a2c5f154a8f8da59be47f9d54db; no voters: 2a06e94e44f24a35b5da6da522938c92
08:03:26.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.568620 16863 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493 [term 2 FOLLOWER]: Leader pre-election won for term 3
08:03:26.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.568723 16863 raft_consensus.cc:491] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:03:26.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.568814 16863 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493 [term 2 FOLLOWER]: Advancing to term 3
08:03:26.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.570868 16863 raft_consensus.cc:513] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2a06e94e44f24a35b5da6da522938c92" member_type: VOTER last_known_addr { host: "127.15.168.190" port: 33711 } } peers { permanent_uuid: "e84d2a2c5f154a8f8da59be47f9d54db" member_type: VOTER last_known_addr { host: "127.15.168.189" port: 43117 } } peers { permanent_uuid: "5bee3c957fe9436399c523d7516cc493" member_type: VOTER last_known_addr { host: "127.15.168.188" port: 38185 } }
08:03:26.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.571545 16863 leader_election.cc:290] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493 [CANDIDATE]: Term 3 election: Requested vote from peers 2a06e94e44f24a35b5da6da522938c92 (127.15.168.190:33711), e84d2a2c5f154a8f8da59be47f9d54db (127.15.168.189:43117)
08:03:26.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.572085 16166 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5bee3c957fe9436399c523d7516cc493" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "e84d2a2c5f154a8f8da59be47f9d54db"
08:03:26.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.572314 16166 raft_consensus.cc:3058] T 00000000000000000000000000000000 P e84d2a2c5f154a8f8da59be47f9d54db [term 2 FOLLOWER]: Advancing to term 3
08:03:26.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:26.572750 16223 leader_election.cc:336] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 2a06e94e44f24a35b5da6da522938c92 (127.15.168.190:33711): Network error: Client connection negotiation failed: client connection to 127.15.168.190:33711: connect: Connection refused (error 111)
08:03:26.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.574512 16166 raft_consensus.cc:2466] T 00000000000000000000000000000000 P e84d2a2c5f154a8f8da59be47f9d54db [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5bee3c957fe9436399c523d7516cc493 in term 3.
08:03:26.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.575024 16225 leader_election.cc:304] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5bee3c957fe9436399c523d7516cc493, e84d2a2c5f154a8f8da59be47f9d54db; no voters: 2a06e94e44f24a35b5da6da522938c92
08:03:26.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.575379 16863 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493 [term 3 FOLLOWER]: Leader election won for term 3
08:03:26.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.575843 16863 raft_consensus.cc:695] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493 [term 3 LEADER]: Becoming Leader. State: Replica: 5bee3c957fe9436399c523d7516cc493, State: Running, Role: LEADER
08:03:26.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.576366 16863 consensus_queue.cc:237] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2a06e94e44f24a35b5da6da522938c92" member_type: VOTER last_known_addr { host: "127.15.168.190" port: 33711 } } peers { permanent_uuid: "e84d2a2c5f154a8f8da59be47f9d54db" member_type: VOTER last_known_addr { host: "127.15.168.189" port: 43117 } } peers { permanent_uuid: "5bee3c957fe9436399c523d7516cc493" member_type: VOTER last_known_addr { host: "127.15.168.188" port: 38185 } }
08:03:26.577 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.15.168.190:33711(127.15.168.190:33711)] connecting to peer
08:03:26.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.577972 16869 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 5bee3c957fe9436399c523d7516cc493. Latest consensus state: current_term: 3 leader_uuid: "5bee3c957fe9436399c523d7516cc493" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2a06e94e44f24a35b5da6da522938c92" member_type: VOTER last_known_addr { host: "127.15.168.190" port: 33711 } } peers { permanent_uuid: "e84d2a2c5f154a8f8da59be47f9d54db" member_type: VOTER last_known_addr { host: "127.15.168.189" port: 43117 } } peers { permanent_uuid: "5bee3c957fe9436399c523d7516cc493" member_type: VOTER last_known_addr { host: "127.15.168.188" port: 38185 } } }
08:03:26.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.578330 16869 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493 [sys.catalog]: This master's current role is: LEADER
08:03:26.579 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711
08:03:26.579 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.168.190:33711(127.15.168.190:33711)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711
08:03:26.579 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.168.190:33711: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711
08:03:26.580 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.168.188:38185,127.15.168.190:33711,127.15.168.189:43117; will retry
08:03:26.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.580816 16870 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
08:03:26.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.583267 16870 catalog_manager.cc:671] Loaded metadata for table kudu_system.kudu_transactions [id=9b2524530e84418583c240c36b4d6d52]
08:03:26.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.583895 16870 catalog_manager.cc:671] Loaded metadata for table txn_manager_fallback [id=c38065584d484ecb87535dca2fe900a1]
08:03:26.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.585739 16870 tablet_loader.cc:96] loaded metadata for tablet e904ce805e75413abce8e621262820b8 (table txn_manager_fallback [id=c38065584d484ecb87535dca2fe900a1])
08:03:26.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.586169 16870 tablet_loader.cc:96] loaded metadata for tablet eec0660dc1864bc9843e00f0f615364e (table kudu_system.kudu_transactions [id=9b2524530e84418583c240c36b4d6d52])
08:03:26.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.586544 16870 tablet_loader.cc:96] loaded metadata for tablet fac3dfebf08c4f418ff9fa5397d50084 (table txn_manager_fallback [id=c38065584d484ecb87535dca2fe900a1])
08:03:26.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.586833 16870 catalog_manager.cc:1486] Initializing Kudu cluster ID...
08:03:26.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.587872 16870 catalog_manager.cc:1261] Loaded cluster ID: c4416adc63694b9a8ae48a3ebc360f91
08:03:26.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.587982 16870 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
08:03:26.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.589152 16870 catalog_manager.cc:1506] Loading token signing keys...
08:03:26.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.590021 16870 catalog_manager.cc:5965] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493: Loaded TSK: 0
08:03:26.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.590893 16870 catalog_manager.cc:1516] Initializing in-progress tserver states...
08:03:26.657 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.15.168.190:33711(127.15.168.190:33711)] connecting to peer
08:03:26.658 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711
08:03:26.658 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.168.190:33711(127.15.168.190:33711)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711
08:03:26.659 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.168.190:33711: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711
08:03:26.661 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
08:03:26.662 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=204), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.188:38185, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.190:33711, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.189:43117, [1ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.190:33711: Network error: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [2ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.188:38185: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.189:43117: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.15.168.188:38185,127.15.168.190:33711,127.15.168.189:43117) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.188:38185, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.190:33711, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.189:43117, [22ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.188:38185: OK, [23ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.190:33711: Network error: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [23ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.189:43117: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.15.168.188:38185,127.15.168.190:33711,127.15.168.189:43117) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.188:38185, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.190:33711, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.189:43117, [42ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.188:38185: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.190:33711: Network error: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [43ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.189:43117: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.15.168.188:38185,127.15.168.190:33711,127.15.168.189:43117) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.188:38185, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.190:33711, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.189:43117, [61ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.190:33711: Network error: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [62ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.188:38185: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.189:43117: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.15.168.188:38185,127.15.168.190:33711,127.15.168.189:43117) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.188:38185, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.190:33711, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.189:43117, [81ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.190:33711: Network error: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [82ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.188:38185: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.189:43117: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.15.168.188:38185,127.15.168.190:33711,127.15.168.189:43117) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.188:38185, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.190:33711, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.189:43117, [101ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.190:33711: Network error: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [101ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.188:38185: OK, [102ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.189:43117: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.15.168.188:38185,127.15.168.190:33711,127.15.168.189:43117) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.188:38185, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.190:33711, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.189:43117, [121ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.188:38185: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.190:33711: Network error: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [122ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.189:43117: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.15.168.188:38185,127.15.168.190:33711,127.15.168.189:43117) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.188:38185, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.190:33711, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.189:43117, [202ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.190:33711: Network error: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [203ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.189:43117: OK, [203ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.188:38185: OK, deferred=Deferred@1235294393(state=PENDING, result=null, callback=(continuation of Deferred@1496766225 after retry RPC after error@891728782) -> (continuation of Deferred@309413072 after retry RPC after error@1406460663) -> (continuation of Deferred@1010121155 after retry RPC after error@1007634184) -> (continuation of Deferred@608224157 after retry RPC after error@1349996216) -> (continuation of Deferred@1159306836 after retry RPC after error@65582840) -> (continuation of Deferred@555139379 after retry RPC after error@1274960167) -> (continuation of Deferred@1109165362 after retry RPC after error@67478783), errback=(continuation of Deferred@1496766225 after retry RPC after error@891728782) -> (continuation of Deferred@309413072 after retry RPC after error@1406460663) -> (continuation of Deferred@1010121155 after retry RPC after error@1007634184) -> (continuation of Deferred@608224157 after retry RPC after error@1349996216) -> (continuation of Deferred@1159306836 after retry RPC after error@65582840) -> (continuation of Deferred@555139379 after retry RPC after error@1274960167) -> (continuation of Deferred@1109165362 after retry RPC after error@67478783))) after lookup
08:03:26.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:26.662236 16223 consensus_peers.cc:487] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493 -> Peer 2a06e94e44f24a35b5da6da522938c92 (127.15.168.190:33711): Couldn't send request to peer 2a06e94e44f24a35b5da6da522938c92. Status: Network error: Client connection negotiation failed: client connection to 127.15.168.190:33711: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
08:03:26.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.706192 16166 raft_consensus.cc:1273] T 00000000000000000000000000000000 P e84d2a2c5f154a8f8da59be47f9d54db [term 3 FOLLOWER]: Refusing update from remote peer 5bee3c957fe9436399c523d7516cc493: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 3 index: 12. (index mismatch)
08:03:26.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:26.706377 16628 heartbeater.cc:646] Failed to heartbeat to 127.15.168.190:33711 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.15.168.190:33711: connect: Connection refused (error 111)
08:03:26.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.707003 16869 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e84d2a2c5f154a8f8da59be47f9d54db" member_type: VOTER last_known_addr { host: "127.15.168.189" port: 43117 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
08:03:26.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:26.709282 16268 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:36328 (request call id 16) took 45 ms (client timeout 45 ms). Trace:
08:03:26.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:26.709470 16268 rpcz_store.cc:269] 0410 08:03:26.663311 (+ 0us) service_pool.cc:168] Inserting onto call queue
08:03:26.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0410 08:03:26.663394 (+ 83us) service_pool.cc:225] Handling call
08:03:26.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0410 08:03:26.709187 (+ 45793us) inbound_call.cc:173] Queueing success response
08:03:26.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"dns_us":135,"init.queue_time_us":475,"thread_start_us":1530,"threads_started":5}
08:03:26.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.712117 16864 sys_catalog.cc:455] T 00000000000000000000000000000000 P e84d2a2c5f154a8f8da59be47f9d54db [sys.catalog]: SysCatalogTable state changed. Reason: New leader 5bee3c957fe9436399c523d7516cc493. Latest consensus state: current_term: 3 leader_uuid: "5bee3c957fe9436399c523d7516cc493" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2a06e94e44f24a35b5da6da522938c92" member_type: VOTER last_known_addr { host: "127.15.168.190" port: 33711 } } peers { permanent_uuid: "e84d2a2c5f154a8f8da59be47f9d54db" member_type: VOTER last_known_addr { host: "127.15.168.189" port: 43117 } } peers { permanent_uuid: "5bee3c957fe9436399c523d7516cc493" member_type: VOTER last_known_addr { host: "127.15.168.188" port: 38185 } } }
08:03:26.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.712404 16864 sys_catalog.cc:458] T 00000000000000000000000000000000 P e84d2a2c5f154a8f8da59be47f9d54db [sys.catalog]: This master's current role is: FOLLOWER
08:03:26.716 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=9, TimeoutTracker(timeout=250, elapsed=258), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.188:38185, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.190:33711, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.189:43117, [1ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.190:33711: Network error: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [2ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.188:38185: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.189:43117: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.15.168.188:38185,127.15.168.190:33711,127.15.168.189:43117) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.188:38185, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.190:33711, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.189:43117, [22ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.188:38185: OK, [23ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.190:33711: Network error: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [23ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.189:43117: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.15.168.188:38185,127.15.168.190:33711,127.15.168.189:43117) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.188:38185, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.190:33711, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.189:43117, [42ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.188:38185: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.190:33711: Network error: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [43ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.189:43117: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.15.168.188:38185,127.15.168.190:33711,127.15.168.189:43117) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.188:38185, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.190:33711, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.189:43117, [61ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.190:33711: Network error: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [62ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.188:38185: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.189:43117: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.15.168.188:38185,127.15.168.190:33711,127.15.168.189:43117) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.188:38185, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.190:33711, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.189:43117, [81ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.190:33711: Network error: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [82ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.188:38185: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.189:43117: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.15.168.188:38185,127.15.168.190:33711,127.15.168.189:43117) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.188:38185, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.190:33711, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.189:43117, [101ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.190:33711: Network error: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [101ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.188:38185: OK, [102ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.189:43117: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.15.168.188:38185,127.15.168.190:33711,127.15.168.189:43117) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.188:38185, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.190:33711, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.189:43117, [121ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.188:38185: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.190:33711: Network error: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [122ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.189:43117: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.15.168.188:38185,127.15.168.190:33711,127.15.168.189:43117) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.188:38185, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.190:33711, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.189:43117, [202ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.190:33711: Network error: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [203ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.189:43117: OK, [203ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.188:38185: OK, [205ms] sending RPC to server master-127.15.168.188:38185, [258ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@1235294393(state=PENDING, result=null, callback=(continuation of Deferred@1496766225 after retry RPC after error@891728782) -> (continuation of Deferred@309413072 after retry RPC after error@1406460663) -> (continuation of Deferred@1010121155 after retry RPC after error@1007634184) -> (continuation of Deferred@608224157 after retry RPC after error@1349996216) -> (continuation of Deferred@1159306836 after retry RPC after error@65582840) -> (continuation of Deferred@555139379 after retry RPC after error@1274960167) -> (continuation of Deferred@1109165362 after retry RPC after error@67478783) -> (continuation of Deferred@284515997 after retry RPC@60591752), errback=(continuation of Deferred@1496766225 after retry RPC after error@891728782) -> (continuation of Deferred@309413072 after retry RPC after error@1406460663) -> (continuation of Deferred@1010121155 after retry RPC after error@1007634184) -> (continuation of Deferred@608224157 after retry RPC after error@1349996216) -> (continuation of Deferred@1159306836 after retry RPC after error@65582840) -> (continuation of Deferred@555139379 after retry RPC after error@1274960167) -> (continuation of Deferred@1109165362 after retry RPC after error@67478783) -> (continuation of Deferred@284515997 after retry RPC@60591752)))
08:03:26.717 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=9, TimeoutTracker(timeout=250, elapsed=258), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.188:38185, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.190:33711, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.189:43117, [1ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.190:33711: Network error: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [2ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.188:38185: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.189:43117: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.15.168.188:38185,127.15.168.190:33711,127.15.168.189:43117) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.188:38185, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.190:33711, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.189:43117, [22ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.188:38185: OK, [23ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.190:33711: Network error: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [23ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.189:43117: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.15.168.188:38185,127.15.168.190:33711,127.15.168.189:43117) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.188:38185, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.190:33711, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.189:43117, [42ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.188:38185: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.190:33711: Network error: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [43ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.189:43117: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.15.168.188:38185,127.15.168.190:33711,127.15.168.189:43117) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.188:38185, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.190:33711, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.189:43117, [61ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.190:33711: Network error: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [62ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.188:38185: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.189:43117: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.15.168.188:38185,127.15.168.190:33711,127.15.168.189:43117) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.188:38185, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.190:33711, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.189:43117, [81ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.190:33711: Network error: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [82ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.188:38185: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.189:43117: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.15.168.188:38185,127.15.168.190:33711,127.15.168.189:43117) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.188:38185, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.190:33711, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.189:43117, [101ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.190:33711: Network error: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [101ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.188:38185: OK, [102ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.189:43117: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.15.168.188:38185,127.15.168.190:33711,127.15.168.189:43117) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.188:38185, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.190:33711, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.189:43117, [121ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.188:38185: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.190:33711: Network error: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [122ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.189:43117: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.15.168.188:38185,127.15.168.190:33711,127.15.168.189:43117) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.188:38185, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.190:33711, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.189:43117, [202ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.190:33711: Network error: Failed to connect to peer master-127.15.168.190:33711(127.15.168.190:33711): Connection refused: /127.15.168.190:33711, [203ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.189:43117: OK, [203ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.188:38185: OK, [205ms] sending RPC to server master-127.15.168.188:38185, [258ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@1235294393(state=PENDING, result=null, callback=(continuation of Deferred@1496766225 after retry RPC after error@891728782) -> (continuation of Deferred@309413072 after retry RPC after error@1406460663) -> (continuation of Deferred@1010121155 after retry RPC after error@1007634184) -> (continuation of Deferred@608224157 after retry RPC after error@1349996216) -> (continuation of Deferred@1159306836 after retry RPC after error@65582840) -> (continuation of Deferred@555139379 after retry RPC after error@1274960167) -> (continuation of Deferred@1109165362 after retry RPC after error@67478783) -> (continuation of Deferred@284515997 after retry RPC@60591752), errback=(continuation of Deferred@1496766225 after retry RPC after error@891728782) -> (continuation of Deferred@309413072 after retry RPC after error@1406460663) -> (continuation of Deferred@1010121155 after retry RPC after error@1007634184) -> (continuation of Deferred@608224157 after retry RPC after error@1349996216) -> (continuation of Deferred@1159306836 after retry RPC after error@65582840) -> (continuation of Deferred@555139379 after retry RPC after error@1274960167) -> (continuation of Deferred@1109165362 after retry RPC after error@67478783) -> (continuation of Deferred@284515997 after retry RPC@60591752)))
08:03:26.717 [DEBUG - pool-38-thread-1] (KuduRpc.java:260) Handling a callback on RPC KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=0, TimeoutTracker(timeout=0, elapsed=0), No traces, deferred=null) with no deferred attached!
08:03:26.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.721307 16237 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:59366:
08:03:26.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
08:03:26.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
08:03:26.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:26.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
08:03:26.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
08:03:26.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:03:26.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:26.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:03:26.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:03:26.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:26.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:26.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:26.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:26.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
08:03:26.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
08:03:26.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:03:26.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:26.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:03:26.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:03:26.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:26.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:26.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:26.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:26.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
08:03:26.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
08:03:26.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:03:26.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:26.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:03:26.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:03:26.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:26.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:26.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:26.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:26.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
08:03:26.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
08:03:26.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
08:03:26.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:26.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:03:26.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:03:26.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:26.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:26.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:26.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:26.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
08:03:26.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
08:03:26.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
08:03:26.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
08:03:26.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:26.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
08:03:26.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
08:03:26.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:26.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
08:03:26.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:26.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:26.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:26.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
08:03:26.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:26.724022 16237 catalog_manager.cc:6943] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
08:03:26.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:26.736428 16487 heartbeater.cc:646] Failed to heartbeat to 127.15.168.190:33711 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.15.168.190:33711: connect: Connection refused (error 111)
08:03:26.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.737496 16864 sys_catalog.cc:455] T 00000000000000000000000000000000 P e84d2a2c5f154a8f8da59be47f9d54db [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "5bee3c957fe9436399c523d7516cc493" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2a06e94e44f24a35b5da6da522938c92" member_type: VOTER last_known_addr { host: "127.15.168.190" port: 33711 } } peers { permanent_uuid: "e84d2a2c5f154a8f8da59be47f9d54db" member_type: VOTER last_known_addr { host: "127.15.168.189" port: 43117 } } peers { permanent_uuid: "5bee3c957fe9436399c523d7516cc493" member_type: VOTER last_known_addr { host: "127.15.168.188" port: 38185 } } }
08:03:26.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.737757 16864 sys_catalog.cc:458] T 00000000000000000000000000000000 P e84d2a2c5f154a8f8da59be47f9d54db [sys.catalog]: This master's current role is: FOLLOWER
08:03:26.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.758230 16863 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "5bee3c957fe9436399c523d7516cc493" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2a06e94e44f24a35b5da6da522938c92" member_type: VOTER last_known_addr { host: "127.15.168.190" port: 33711 } } peers { permanent_uuid: "e84d2a2c5f154a8f8da59be47f9d54db" member_type: VOTER last_known_addr { host: "127.15.168.189" port: 43117 } } peers { permanent_uuid: "5bee3c957fe9436399c523d7516cc493" member_type: VOTER last_known_addr { host: "127.15.168.188" port: 38185 } } }
08:03:26.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.758549 16863 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493 [sys.catalog]: This master's current role is: LEADER
08:03:26.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:26.760890 16654 txn_status_manager.cc:1391] automatically aborted stale txn (ID 0) past 1.003s from last keepalive heartbeat (effective timeout is 1.000s)
08:03:26.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:26.777181 16694 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.15.168.190:33711)
08:03:26.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:26.781118 16881 tablet_replica.cc:1304] Aborted: operation has been aborted: cancelling pending write operations
08:03:26.785 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) terminating keepalive task (txn ID 0) due to exception org.apache.kudu.client.NonRecoverableException: Failed to write to server: 82c7f3b9c7774a5281eae28fb7e64216 (127.15.168.131:40597): transaction ID 0 is not available for further commits: state: ABORT_IN_PROGRESS user: "slave" start_timestamp: 1744272205 last_transition_timestamp: 1744272206
08:03:27.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:27.119160 16223 consensus_peers.cc:487] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493 -> Peer 2a06e94e44f24a35b5da6da522938c92 (127.15.168.190:33711): Couldn't send request to peer 2a06e94e44f24a35b5da6da522938c92. Status: Network error: Client connection negotiation failed: client connection to 127.15.168.190:33711: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
08:03:27.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:27.486642 16205 master_runner.cc:186] Error getting master registration for 127.15.168.190:33711: Network error: Client connection negotiation failed: client connection to 127.15.168.190:33711: connect: Connection refused (error 111),
08:03:27.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:27.486857 16205 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:03:27.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:27.611034 16223 consensus_peers.cc:487] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493 -> Peer 2a06e94e44f24a35b5da6da522938c92 (127.15.168.190:33711): Couldn't send request to peer 2a06e94e44f24a35b5da6da522938c92. Status: Network error: Client connection negotiation failed: client connection to 127.15.168.190:33711: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
08:03:27.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:27.699930 16627 heartbeater.cc:499] Master 127.15.168.188:38185 was elected leader, sending a full tablet report...
08:03:27.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:27.714175 16486 heartbeater.cc:499] Master 127.15.168.188:38185 was elected leader, sending a full tablet report...
08:03:27.928 [DEBUG - main] (Connection.java:575) [peer master-127.15.168.188:38185(127.15.168.188:38185)] disconnecting while in state READY
08:03:27.928 [DEBUG - main] (Connection.java:575) [peer 82c7f3b9c7774a5281eae28fb7e64216(127.15.168.131:40597)] disconnecting while in state READY
08:03:27.928 [DEBUG - main] (Connection.java:575) [peer master-127.15.168.189:43117(127.15.168.189:43117)] disconnecting while in state READY
08:03:27.928 [DEBUG - main] (Connection.java:575) [peer master-127.15.168.190:33711(127.15.168.190:33711)] disconnecting while in state TERMINATED
08:03:27.929 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 82c7f3b9c7774a5281eae28fb7e64216(127.15.168.131:40597)] handling channelInactive
08:03:27.929 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 82c7f3b9c7774a5281eae28fb7e64216(127.15.168.131:40597)] cleaning up while in state READY due to: connection closed
08:03:27.929 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
08:03:27.930 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.15.168.188:38185(127.15.168.188:38185)] handling channelInactive
08:03:27.930 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.168.188:38185(127.15.168.188:38185)] cleaning up while in state READY due to: connection closed
08:03:27.930 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.15.168.189:43117(127.15.168.189:43117)] handling channelInactive
08:03:27.930 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.168.189:43117(127.15.168.189:43117)] cleaning up while in state READY due to: connection closed
08:03:27.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:27.930505 16034 external_mini_cluster.cc:1661] Attempting to check leaks for /tmp/dist-test-taskbKfKhV/build/asan/bin/kudu pid 16334
08:03:27.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:27.938642 16481 generic_service.cc:196] Checking for leaks (request via RPC)
08:03:27.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:27.982872 16769 heartbeater.cc:499] Master 127.15.168.188:38185 was elected leader, sending a full tablet report...
08:03:28.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:28.035113 16034 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskbKfKhV/build/asan/bin/kudu with pid 16334
08:03:28.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:28.053810 16034 external_mini_cluster.cc:1661] Attempting to check leaks for /tmp/dist-test-taskbKfKhV/build/asan/bin/kudu pid 16491
08:03:28.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:28.060774 16622 generic_service.cc:196] Checking for leaks (request via RPC)
08:03:28.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250410 08:03:28.090939 16223 consensus_peers.cc:487] T 00000000000000000000000000000000 P 5bee3c957fe9436399c523d7516cc493 -> Peer 2a06e94e44f24a35b5da6da522938c92 (127.15.168.190:33711): Couldn't send request to peer 2a06e94e44f24a35b5da6da522938c92. Status: Network error: Client connection negotiation failed: client connection to 127.15.168.190:33711: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
08:03:28.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:28.155387 16034 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskbKfKhV/build/asan/bin/kudu with pid 16491
08:03:28.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:28.173265 16034 external_mini_cluster.cc:1661] Attempting to check leaks for /tmp/dist-test-taskbKfKhV/build/asan/bin/kudu pid 16632
08:03:28.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:28.180377 16764 generic_service.cc:196] Checking for leaks (request via RPC)
08:03:28.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:28.302080 16034 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskbKfKhV/build/asan/bin/kudu with pid 16632
08:03:28.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:28.324043 16034 external_mini_cluster.cc:1661] Attempting to check leaks for /tmp/dist-test-taskbKfKhV/build/asan/bin/kudu pid 16123
08:03:28.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:28.325192 16197 generic_service.cc:196] Checking for leaks (request via RPC)
08:03:28.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:28.436960 16034 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskbKfKhV/build/asan/bin/kudu with pid 16123
08:03:28.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:28.457861 16034 external_mini_cluster.cc:1661] Attempting to check leaks for /tmp/dist-test-taskbKfKhV/build/asan/bin/kudu pid 16205
08:03:28.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:28.458956 16278 generic_service.cc:196] Checking for leaks (request via RPC)
08:03:28.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250410 08:03:28.571807 16034 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskbKfKhV/build/asan/bin/kudu with pid 16205
08:03:28.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-04-10T08:03:28Z chronyd exiting
08:03:28.723 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveSwitchesToOtherTxnManager: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Failed to write to server: 82c7f3b9c7774a5281eae28fb7e64216 (127.15.168.131:40597): transaction ID 0 is not open: state: ABORTED user: "slave" start_timestamp: 1744272205 last_transition_timestamp: 1744272206
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduTransaction.doCommitTransaction(KuduTransaction.java:491) ~[main/:?]
at org.apache.kudu.client.KuduTransaction.commitWithMode(KuduTransaction.java:537) ~[main/:?]
at org.apache.kudu.client.KuduTransaction.commit(KuduTransaction.java:284) ~[main/:?]
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveSwitchesToOtherTxnManager(TestKuduTransaction.java:1152) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.RpcProxy.dispatchTxnManagerError(RpcProxy.java:439) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:298) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1503) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1366) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1415) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Failed to write to server: 82c7f3b9c7774a5281eae28fb7e64216 (127.15.168.131:40597): transaction ID 0 is not open: state: ABORTED user: "slave" start_timestamp: 1744272205 last_transition_timestamp: 1744272206
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduTransaction.doCommitTransaction(KuduTransaction.java:491)
at org.apache.kudu.client.KuduTransaction.commitWithMode(KuduTransaction.java:537)
at org.apache.kudu.client.KuduTransaction.commit(KuduTransaction.java:284)
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveSwitchesToOtherTxnManager(TestKuduTransaction.java:1152)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.RpcProxy.dispatchTxnManagerError(RpcProxy.java:439)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:298)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1503)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1366)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1415)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more