15:06:27.639 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
15:06:27.639 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task7b0YEV/test-tmp
15:06:27.639 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/debug/bin
15:06:27.639 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task7b0YEV/java/build/dist-test/../../../build/debug/bin/kudu, test, mini_cluster, --serialization=pb]
15:06:27.641 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586"
extra_master_flags: "--txn_manager_enabled"
extra_master_flags: "--raft_heartbeat_interval_ms=100"
extra_tserver_flags: "--txn_keepalive_interval_ms=1000"
extra_tserver_flags: "--txn_staleness_tracker_interval_ms=250"
extra_tserver_flags: "--enable_txn_system_client_init=true"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
15:06:27.744 [DEBUG - main] (MiniKuduCluster.java:195) Response:
15:06:27.744 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
15:06:27.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:27.745456 30527 mini_oidc.cc:150] Starting JWKS server
15:06:27.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:27.747193 30527 webserver.cc:458] Webserver started at http://127.0.0.1:45165/ using document root <none> and password file <none>
15:06:27.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:27.747340 30527 mini_oidc.cc:186] Starting OIDC Discovery server
15:06:27.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:27.747574 30527 webserver.cc:458] Webserver started at http://127.0.0.1:39811/ using document root <none> and password file <none>
15:06:27.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-12T15:06:27Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
15:06:27.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-12T15:06:27Z Disabled control of system clock
15:06:27.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:27.762351 30527 external_mini_cluster.cc:1351] Running /tmp/dist-test-task7b0YEV/build/debug/bin/kudu
15:06:27.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task7b0YEV/build/debug/bin/kudu
15:06:27.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-0/wal
15:06:27.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-0/data
15:06:27.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:06:27.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:06:27.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:06:27.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:27.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:27.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:27.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-0/logs
15:06:27.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-0/data/info.pb
15:06:27.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:27.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:06:27.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:06:27.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:06:27.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:06:27.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:27.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:27.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:27.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:06:27.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:06:27.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:06:27.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:06:27.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.207.254:42611
15:06:27.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.207.254
15:06:27.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:06:27.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.207.212:43549
15:06:27.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:27.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:27.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:27.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:06:27.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.207.254:42611,127.29.207.253:44061,127.29.207.252:42085
15:06:27.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:06:27.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45165/jwks.json
15:06:27.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
15:06:27.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
15:06:27.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:27.865033 30538 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:06:27.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:27.865289 30538 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:06:27.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:27.865353 30538 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:06:27.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:27.868417 30538 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:06:27.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:27.868487 30538 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
15:06:27.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:27.868515 30538 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:06:27.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:27.868539 30538 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:06:27.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:27.868559 30538 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:06:27.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:27.868578 30538 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:06:27.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:27.868595 30538 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45165/jwks.json
15:06:27.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:27.872283 30538 master_runner.cc:386] Master server non-default flags:
15:06:27.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:27.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.207.212:43549
15:06:27.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:27.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:27.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
15:06:27.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-0/data
15:06:27.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-0/wal
15:06:27.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:06:27.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.207.254:42611,127.29.207.253:44061,127.29.207.252:42085
15:06:27.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
15:06:27.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:27.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:27.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:06:27.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.207.254:42611
15:06:27.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:06:27.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:06:27.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:06:27.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45165/jwks.json
15:06:27.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:27.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:27.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-0/data/info.pb
15:06:27.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.207.254
15:06:27.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:06:27.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:06:27.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.30538
15:06:27.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:27.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:06:27.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:06:27.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:27.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-0/logs
15:06:27.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:27.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:06:27.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:06:27.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:06:27.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:06:27.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:06:27.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
15:06:27.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Jan 2025 14:43:14 UTC on 24a791456cd2
15:06:27.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4141
15:06:27.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:27.873598 30538 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:06:27.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:27.874837 30538 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:06:27.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:27.881340 30546 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:27.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:27.881358 30544 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:27.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:27.881414 30543 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:27.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:27.881501 30538 server_base.cc:1034] running on GCE node
15:06:27.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:27.882174 30538 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:06:27.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:27.883147 30538 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:06:27.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:27.884310 30538 hybrid_clock.cc:648] HybridClock initialized: now 1736694387884297 us; error 30 us; skew 500 ppm
15:06:27.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:27.886257 30538 webserver.cc:458] Webserver started at http://127.29.207.254:40757/ using document root <none> and password file <none>
15:06:27.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:27.886869 30538 fs_manager.cc:362] Metadata directory not provided
15:06:27.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:27.886929 30538 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:06:27.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:27.887128 30538 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:06:27.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:27.888741 30538 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-0/data/instance:
15:06:27.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1a86734036df442d9067f984db9f965e"
15:06:27.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:06:27 on dist-test-slave-xnn2"
15:06:27.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:27.889212 30538 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-0/wal/instance:
15:06:27.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1a86734036df442d9067f984db9f965e"
15:06:27.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:06:27 on dist-test-slave-xnn2"
15:06:27.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:27.892452 30538 fs_manager.cc:696] Time spent creating directory manager: real 0.003s user 0.005s sys 0.000s
15:06:27.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:27.894412 30552 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:06:27.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:27.895344 30538 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
15:06:27.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:27.895452 30538 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-0/data,/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-0/wal
15:06:27.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1a86734036df442d9067f984db9f965e"
15:06:27.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:06:27 on dist-test-slave-xnn2"
15:06:27.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:27.895550 30538 fs_report.cc:389] FS layout report
15:06:27.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:06:27.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-0/wal
15:06:27.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-0/wal
15:06:27.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-0/data/data
15:06:27.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:06:27.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:06:27.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:06:27.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:06:27.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:06:27.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:06:27.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:06:27.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:06:27.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:06:27.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:06:27.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:06:27.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:27.908146 30538 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:06:27.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:27.908830 30538 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:06:27.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:27.908990 30538 kserver.cc:163] Server-wide thread pool size limit: 3276
15:06:27.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:27.916033 30614 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.207.254:42611 every 8 connection(s)
15:06:27.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:27.916035 30538 rpc_server.cc:307] RPC server started. Bound to: 127.29.207.254:42611
15:06:27.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:27.917109 30538 server_base.cc:1166] Dumped server information to /tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-0/data/info.pb
15:06:27.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:27.917830 30527 external_mini_cluster.cc:1413] Started /tmp/dist-test-task7b0YEV/build/debug/bin/kudu as pid 30538
15:06:27.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:27.917955 30527 external_mini_cluster.cc:1427] Reading /tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-0/wal/instance
15:06:27.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:27.919399 30527 external_mini_cluster.cc:1351] Running /tmp/dist-test-task7b0YEV/build/debug/bin/kudu
15:06:27.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task7b0YEV/build/debug/bin/kudu
15:06:27.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-1/wal
15:06:27.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-1/data
15:06:27.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:06:27.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:06:27.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:06:27.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:27.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:27.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:27.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-1/logs
15:06:27.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-1/data/info.pb
15:06:27.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:27.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:06:27.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:06:27.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:06:27.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:06:27.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:27.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:27.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:27.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:06:27.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:06:27.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:06:27.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:06:27.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.207.253:44061
15:06:27.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.207.253
15:06:27.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:06:27.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.207.212:43549
15:06:27.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:27.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:27.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:27.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:06:27.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.207.254:42611,127.29.207.253:44061,127.29.207.252:42085
15:06:27.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:06:27.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45165/jwks.json
15:06:27.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
15:06:27.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
15:06:27.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:27.919742 30615 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:06:27.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:27.923745 30615 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.207.254" port: 42611 } has no permanent_uuid. Determining permanent_uuid...
15:06:27.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:27.935051 30615 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.207.253" port: 44061 } has no permanent_uuid. Determining permanent_uuid...
15:06:27.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:27.935856 30556 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.207.253:44061: connect: Connection refused (error 111)
15:06:27.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:27.936728 30615 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.207.253:44061: Network error: Client connection negotiation failed: client connection to 127.29.207.253:44061: connect: Connection refused (error 111)
15:06:27.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:27.986006 30615 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.207.253" port: 44061 } attempt: 1
15:06:27.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:27.987581 30615 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.207.253:44061: Network error: Client connection negotiation failed: client connection to 127.29.207.253:44061: connect: Connection refused (error 111)
15:06:28.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.028604 30617 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:06:28.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.028839 30617 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:06:28.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.028887 30617 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:06:28.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.031975 30617 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:06:28.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.032038 30617 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
15:06:28.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.032055 30617 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:06:28.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.032073 30617 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:06:28.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.032091 30617 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:06:28.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.032109 30617 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:06:28.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.032123 30617 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45165/jwks.json
15:06:28.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.035815 30617 master_runner.cc:386] Master server non-default flags:
15:06:28.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:28.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.207.212:43549
15:06:28.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:28.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:28.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
15:06:28.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-1/data
15:06:28.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-1/wal
15:06:28.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:06:28.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.207.254:42611,127.29.207.253:44061,127.29.207.252:42085
15:06:28.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
15:06:28.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:28.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:28.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:06:28.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.207.253:44061
15:06:28.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:06:28.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:06:28.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:06:28.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45165/jwks.json
15:06:28.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:28.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:28.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-1/data/info.pb
15:06:28.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.207.253
15:06:28.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:06:28.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:06:28.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.30617
15:06:28.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:28.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:06:28.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:06:28.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:28.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-1/logs
15:06:28.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:28.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:06:28.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:06:28.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:06:28.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:06:28.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:06:28.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
15:06:28.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Jan 2025 14:43:14 UTC on 24a791456cd2
15:06:28.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4141
15:06:28.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.036900 30617 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:06:28.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.038004 30617 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:06:28.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.044745 30626 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:28.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.044754 30628 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:28.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.044754 30625 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:28.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.045012 30617 server_base.cc:1034] running on GCE node
15:06:28.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.045683 30617 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:06:28.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.046645 30617 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:06:28.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.047806 30617 hybrid_clock.cc:648] HybridClock initialized: now 1736694388047793 us; error 38 us; skew 500 ppm
15:06:28.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.049670 30617 webserver.cc:458] Webserver started at http://127.29.207.253:34259/ using document root <none> and password file <none>
15:06:28.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.050251 30617 fs_manager.cc:362] Metadata directory not provided
15:06:28.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.050309 30617 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:06:28.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.050535 30617 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:06:28.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.052158 30617 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-1/data/instance:
15:06:28.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115"
15:06:28.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:06:28 on dist-test-slave-xnn2"
15:06:28.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.052623 30617 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-1/wal/instance:
15:06:28.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115"
15:06:28.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:06:28 on dist-test-slave-xnn2"
15:06:28.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.055783 30615 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.207.253" port: 44061 } attempt: 2
15:06:28.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.055955 30617 fs_manager.cc:696] Time spent creating directory manager: real 0.003s user 0.000s sys 0.004s
15:06:28.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.057030 30615 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.207.253:44061: Network error: Client connection negotiation failed: client connection to 127.29.207.253:44061: connect: Connection refused (error 111)
15:06:28.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.058173 30634 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:06:28.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.059228 30617 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
15:06:28.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.059330 30617 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-1/data,/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-1/wal
15:06:28.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115"
15:06:28.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:06:28 on dist-test-slave-xnn2"
15:06:28.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.059404 30617 fs_report.cc:389] FS layout report
15:06:28.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:06:28.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-1/wal
15:06:28.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-1/wal
15:06:28.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-1/data/data
15:06:28.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:06:28.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:06:28.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:06:28.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:06:28.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:06:28.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:06:28.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:06:28.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:06:28.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:06:28.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:06:28.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:06:28.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.071976 30617 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:06:28.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.072613 30617 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:06:28.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.072749 30617 kserver.cc:163] Server-wide thread pool size limit: 3276
15:06:28.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.080503 30617 rpc_server.cc:307] RPC server started. Bound to: 127.29.207.253:44061
15:06:28.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.080513 30696 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.207.253:44061 every 8 connection(s)
15:06:28.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.081496 30617 server_base.cc:1166] Dumped server information to /tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-1/data/info.pb
15:06:28.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.084033 30697 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:06:28.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.085693 30527 external_mini_cluster.cc:1413] Started /tmp/dist-test-task7b0YEV/build/debug/bin/kudu as pid 30617
15:06:28.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.085798 30527 external_mini_cluster.cc:1427] Reading /tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-1/wal/instance
15:06:28.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.086540 30527 external_mini_cluster.cc:1351] Running /tmp/dist-test-task7b0YEV/build/debug/bin/kudu
15:06:28.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task7b0YEV/build/debug/bin/kudu
15:06:28.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-2/wal
15:06:28.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-2/data
15:06:28.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:06:28.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:06:28.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:06:28.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:28.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:28.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:28.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-2/logs
15:06:28.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-2/data/info.pb
15:06:28.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:28.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:06:28.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:06:28.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:06:28.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:06:28.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:28.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:28.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:28.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:06:28.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:06:28.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:06:28.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:06:28.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.207.252:42085
15:06:28.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.207.252
15:06:28.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:06:28.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.207.212:43549
15:06:28.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:28.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:28.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:28.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:06:28.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.207.254:42611,127.29.207.253:44061,127.29.207.252:42085
15:06:28.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:06:28.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45165/jwks.json
15:06:28.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
15:06:28.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
15:06:28.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.087848 30697 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.207.254" port: 42611 } has no permanent_uuid. Determining permanent_uuid...
15:06:28.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.096999 30697 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.207.253" port: 44061 } has no permanent_uuid. Determining permanent_uuid...
15:06:28.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.104558 30697 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.207.252" port: 42085 } has no permanent_uuid. Determining permanent_uuid...
15:06:28.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.105348 30637 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.207.252:42085: connect: Connection refused (error 111)
15:06:28.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.106261 30697 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.207.252:42085: Network error: Client connection negotiation failed: client connection to 127.29.207.252:42085: connect: Connection refused (error 111)
15:06:28.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.148239 30615 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.207.253" port: 44061 } attempt: 3
15:06:28.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.152570 30615 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.207.252" port: 42085 } has no permanent_uuid. Determining permanent_uuid...
15:06:28.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.153856 30615 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.207.252:42085: Network error: Client connection negotiation failed: client connection to 127.29.207.252:42085: connect: Connection refused (error 111)
15:06:28.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.155480 30697 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.207.252" port: 42085 } attempt: 1
15:06:28.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.156716 30697 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.207.252:42085: Network error: Client connection negotiation failed: client connection to 127.29.207.252:42085: connect: Connection refused (error 111)
15:06:28.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.185042 30615 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.207.252" port: 42085 } attempt: 1
15:06:28.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.186633 30615 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.207.252:42085: Network error: Client connection negotiation failed: client connection to 127.29.207.252:42085: connect: Connection refused (error 111)
15:06:28.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.197276 30699 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:06:28.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.197525 30699 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:06:28.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.197571 30699 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:06:28.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.200820 30699 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:06:28.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.200893 30699 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
15:06:28.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.200910 30699 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:06:28.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.200927 30699 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:06:28.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.200945 30699 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:06:28.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.200963 30699 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:06:28.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.200977 30699 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45165/jwks.json
15:06:28.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.204682 30699 master_runner.cc:386] Master server non-default flags:
15:06:28.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:28.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.207.212:43549
15:06:28.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:28.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:28.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
15:06:28.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-2/data
15:06:28.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-2/wal
15:06:28.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:06:28.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.207.254:42611,127.29.207.253:44061,127.29.207.252:42085
15:06:28.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
15:06:28.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:28.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:28.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:06:28.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.207.252:42085
15:06:28.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:06:28.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:06:28.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:06:28.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45165/jwks.json
15:06:28.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:28.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:28.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-2/data/info.pb
15:06:28.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.207.252
15:06:28.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:06:28.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:06:28.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.30699
15:06:28.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:28.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:06:28.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:06:28.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:28.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-2/logs
15:06:28.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:28.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:06:28.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:06:28.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:06:28.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:06:28.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:06:28.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
15:06:28.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Jan 2025 14:43:14 UTC on 24a791456cd2
15:06:28.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4141
15:06:28.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.205803 30699 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:06:28.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.206985 30699 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:06:28.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.213699 30707 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:28.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.213687 30710 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:28.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.213739 30708 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:28.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.213894 30699 server_base.cc:1034] running on GCE node
15:06:28.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.214787 30699 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:06:28.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.215754 30699 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:06:28.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.216930 30699 hybrid_clock.cc:648] HybridClock initialized: now 1736694388216902 us; error 27 us; skew 500 ppm
15:06:28.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.218999 30699 webserver.cc:458] Webserver started at http://127.29.207.252:37191/ using document root <none> and password file <none>
15:06:28.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.219651 30699 fs_manager.cc:362] Metadata directory not provided
15:06:28.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.219709 30699 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:06:28.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.219864 30699 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:06:28.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.221415 30699 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-2/data/instance:
15:06:28.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5168bd8e804e45abae06fee65e7c306d"
15:06:28.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:06:28 on dist-test-slave-xnn2"
15:06:28.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.221870 30699 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-2/wal/instance:
15:06:28.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5168bd8e804e45abae06fee65e7c306d"
15:06:28.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:06:28 on dist-test-slave-xnn2"
15:06:28.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.224871 30697 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.207.252" port: 42085 } attempt: 2
15:06:28.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.225292 30699 fs_manager.cc:696] Time spent creating directory manager: real 0.003s user 0.002s sys 0.004s
15:06:28.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.226204 30697 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.207.252:42085: Network error: Client connection negotiation failed: client connection to 127.29.207.252:42085: connect: Connection refused (error 111)
15:06:28.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.227612 30717 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:06:28.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.228577 30699 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.001s sys 0.000s
15:06:28.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.228705 30699 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-2/data,/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-2/wal
15:06:28.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5168bd8e804e45abae06fee65e7c306d"
15:06:28.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:06:28 on dist-test-slave-xnn2"
15:06:28.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.228808 30699 fs_report.cc:389] FS layout report
15:06:28.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:06:28.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-2/wal
15:06:28.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-2/wal
15:06:28.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-2/data/data
15:06:28.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:06:28.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:06:28.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:06:28.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:06:28.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:06:28.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:06:28.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:06:28.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:06:28.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:06:28.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:06:28.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:06:28.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.244117 30699 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:06:28.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.244771 30699 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:06:28.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.244926 30699 kserver.cc:163] Server-wide thread pool size limit: 3276
15:06:28.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.252275 30699 rpc_server.cc:307] RPC server started. Bound to: 127.29.207.252:42085
15:06:28.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.252282 30779 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.207.252:42085 every 8 connection(s)
15:06:28.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.253300 30699 server_base.cc:1166] Dumped server information to /tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-2/data/info.pb
15:06:28.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.255930 30780 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:06:28.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.258978 30780 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.207.254" port: 42611 } has no permanent_uuid. Determining permanent_uuid...
15:06:28.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.261827 30615 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.207.252" port: 42085 } attempt: 2
15:06:28.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.262490 30527 external_mini_cluster.cc:1413] Started /tmp/dist-test-task7b0YEV/build/debug/bin/kudu as pid 30699
15:06:28.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.262615 30527 external_mini_cluster.cc:1427] Reading /tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-2/wal/instance
15:06:28.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.268529 30780 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.207.253" port: 44061 } has no permanent_uuid. Determining permanent_uuid...
15:06:28.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.271207 30615 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e: Bootstrap starting.
15:06:28.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.274338 30615 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e: Neither blocks nor log segments found. Creating new log.
15:06:28.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.274959 30780 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.207.252" port: 42085 } has no permanent_uuid. Determining permanent_uuid...
15:06:28.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.275203 30615 log.cc:826] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e: Log is configured to *not* fsync() on all Append() calls
15:06:28.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.277276 30615 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e: No bootstrap required, opened a new log
15:06:28.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.281106 30615 raft_consensus.cc:357] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a86734036df442d9067f984db9f965e" member_type: VOTER last_known_addr { host: "127.29.207.254" port: 42611 } } peers { permanent_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115" member_type: VOTER last_known_addr { host: "127.29.207.253" port: 44061 } } peers { permanent_uuid: "5168bd8e804e45abae06fee65e7c306d" member_type: VOTER last_known_addr { host: "127.29.207.252" port: 42085 } }
15:06:28.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.281282 30615 raft_consensus.cc:383] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:06:28.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.281339 30615 raft_consensus.cc:738] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1a86734036df442d9067f984db9f965e, State: Initialized, Role: FOLLOWER
15:06:28.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.281400 30780 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d: Bootstrap starting.
15:06:28.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.281792 30615 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a86734036df442d9067f984db9f965e" member_type: VOTER last_known_addr { host: "127.29.207.254" port: 42611 } } peers { permanent_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115" member_type: VOTER last_known_addr { host: "127.29.207.253" port: 44061 } } peers { permanent_uuid: "5168bd8e804e45abae06fee65e7c306d" member_type: VOTER last_known_addr { host: "127.29.207.252" port: 42085 } }
15:06:28.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.282276 30789 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a86734036df442d9067f984db9f965e" member_type: VOTER last_known_addr { host: "127.29.207.254" port: 42611 } } peers { permanent_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115" member_type: VOTER last_known_addr { host: "127.29.207.253" port: 44061 } } peers { permanent_uuid: "5168bd8e804e45abae06fee65e7c306d" member_type: VOTER last_known_addr { host: "127.29.207.252" port: 42085 } } }
15:06:28.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.282562 30789 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e [sys.catalog]: This master's current role is: FOLLOWER
15:06:28.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.283159 30615 sys_catalog.cc:564] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e [sys.catalog]: configured and running, proceeding with master startup.
15:06:28.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.284236 30780 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d: Neither blocks nor log segments found. Creating new log.
15:06:28.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.284952 30780 log.cc:826] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d: Log is configured to *not* fsync() on all Append() calls
15:06:28.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.286852 30780 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d: No bootstrap required, opened a new log
15:06:28.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.288213 30801 catalog_manager.cc:1559] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:06:28.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.288376 30801 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:06:28.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.287880 30538 master_runner.cc:186] Error getting master registration for 127.29.207.253:44061: OK, instance_id { permanent_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115" instance_seqno: 1736694388078161 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:06:28.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.290170 30538 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:06:28.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.290189 30780 raft_consensus.cc:357] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a86734036df442d9067f984db9f965e" member_type: VOTER last_known_addr { host: "127.29.207.254" port: 42611 } } peers { permanent_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115" member_type: VOTER last_known_addr { host: "127.29.207.253" port: 44061 } } peers { permanent_uuid: "5168bd8e804e45abae06fee65e7c306d" member_type: VOTER last_known_addr { host: "127.29.207.252" port: 42085 } }
15:06:28.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.290360 30780 raft_consensus.cc:383] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:06:28.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.290419 30780 raft_consensus.cc:738] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5168bd8e804e45abae06fee65e7c306d, State: Initialized, Role: FOLLOWER
15:06:28.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.290964 30780 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a86734036df442d9067f984db9f965e" member_type: VOTER last_known_addr { host: "127.29.207.254" port: 42611 } } peers { permanent_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115" member_type: VOTER last_known_addr { host: "127.29.207.253" port: 44061 } } peers { permanent_uuid: "5168bd8e804e45abae06fee65e7c306d" member_type: VOTER last_known_addr { host: "127.29.207.252" port: 42085 } }
15:06:28.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.292001 30803 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a86734036df442d9067f984db9f965e" member_type: VOTER last_known_addr { host: "127.29.207.254" port: 42611 } } peers { permanent_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115" member_type: VOTER last_known_addr { host: "127.29.207.253" port: 44061 } } peers { permanent_uuid: "5168bd8e804e45abae06fee65e7c306d" member_type: VOTER last_known_addr { host: "127.29.207.252" port: 42085 } } }
15:06:28.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.292161 30803 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [sys.catalog]: This master's current role is: FOLLOWER
15:06:28.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.292312 30780 sys_catalog.cc:564] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [sys.catalog]: configured and running, proceeding with master startup.
15:06:28.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.297978 30814 catalog_manager.cc:1559] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:06:28.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.298216 30814 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:06:28.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.298322 30699 master_runner.cc:186] Error getting master registration for 127.29.207.253:44061: OK, instance_id { permanent_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115" instance_seqno: 1736694388078161 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:06:28.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.301393 30699 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:06:28.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.303603 30803 raft_consensus.cc:491] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:06:28.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.303774 30803 raft_consensus.cc:513] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a86734036df442d9067f984db9f965e" member_type: VOTER last_known_addr { host: "127.29.207.254" port: 42611 } } peers { permanent_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115" member_type: VOTER last_known_addr { host: "127.29.207.253" port: 44061 } } peers { permanent_uuid: "5168bd8e804e45abae06fee65e7c306d" member_type: VOTER last_known_addr { host: "127.29.207.252" port: 42085 } }
15:06:28.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.304796 30803 leader_election.cc:290] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1a86734036df442d9067f984db9f965e (127.29.207.254:42611), 80c9fc0ab13d44ffa2a33e3b8c32f115 (127.29.207.253:44061)
15:06:28.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.305399 30579 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5168bd8e804e45abae06fee65e7c306d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1a86734036df442d9067f984db9f965e" is_pre_election: true
15:06:28.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.305701 30579 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5168bd8e804e45abae06fee65e7c306d in term 0.
15:06:28.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.306136 30720 leader_election.cc:304] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1a86734036df442d9067f984db9f965e, 5168bd8e804e45abae06fee65e7c306d; no voters:
15:06:28.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.306406 30803 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [term 0 FOLLOWER]: Leader pre-election won for term 1
15:06:28.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.306491 30803 raft_consensus.cc:491] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
15:06:28.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.306571 30803 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [term 0 FOLLOWER]: Advancing to term 1
15:06:28.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.307688 30803 raft_consensus.cc:513] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a86734036df442d9067f984db9f965e" member_type: VOTER last_known_addr { host: "127.29.207.254" port: 42611 } } peers { permanent_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115" member_type: VOTER last_known_addr { host: "127.29.207.253" port: 44061 } } peers { permanent_uuid: "5168bd8e804e45abae06fee65e7c306d" member_type: VOTER last_known_addr { host: "127.29.207.252" port: 42085 } }
15:06:28.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.308076 30803 leader_election.cc:290] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [CANDIDATE]: Term 1 election: Requested vote from peers 1a86734036df442d9067f984db9f965e (127.29.207.254:42611), 80c9fc0ab13d44ffa2a33e3b8c32f115 (127.29.207.253:44061)
15:06:28.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.309041 30579 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5168bd8e804e45abae06fee65e7c306d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1a86734036df442d9067f984db9f965e"
15:06:28.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.309157 30579 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e [term 0 FOLLOWER]: Advancing to term 1
15:06:28.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.310237 30579 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5168bd8e804e45abae06fee65e7c306d in term 1.
15:06:28.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.310573 30720 leader_election.cc:304] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1a86734036df442d9067f984db9f965e, 5168bd8e804e45abae06fee65e7c306d; no voters:
15:06:28.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.312007 30803 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [term 1 FOLLOWER]: Leader election won for term 1
15:06:28.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.312247 30803 raft_consensus.cc:695] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [term 1 LEADER]: Becoming Leader. State: Replica: 5168bd8e804e45abae06fee65e7c306d, State: Running, Role: LEADER
15:06:28.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.312752 30803 consensus_queue.cc:237] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a86734036df442d9067f984db9f965e" member_type: VOTER last_known_addr { host: "127.29.207.254" port: 42611 } } peers { permanent_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115" member_type: VOTER last_known_addr { host: "127.29.207.253" port: 44061 } } peers { permanent_uuid: "5168bd8e804e45abae06fee65e7c306d" member_type: VOTER last_known_addr { host: "127.29.207.252" port: 42085 } }
15:06:28.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.314862 30816 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [sys.catalog]: SysCatalogTable state changed. Reason: New leader 5168bd8e804e45abae06fee65e7c306d. Latest consensus state: current_term: 1 leader_uuid: "5168bd8e804e45abae06fee65e7c306d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a86734036df442d9067f984db9f965e" member_type: VOTER last_known_addr { host: "127.29.207.254" port: 42611 } } peers { permanent_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115" member_type: VOTER last_known_addr { host: "127.29.207.253" port: 44061 } } peers { permanent_uuid: "5168bd8e804e45abae06fee65e7c306d" member_type: VOTER last_known_addr { host: "127.29.207.252" port: 42085 } } }
15:06:28.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.314998 30816 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [sys.catalog]: This master's current role is: LEADER
15:06:28.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.315523 30818 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
15:06:28.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.316601 30818 catalog_manager.cc:1485] Initializing Kudu cluster ID...
15:06:28.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.317342 30697 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.207.252" port: 42085 } attempt: 3
15:06:28.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.323899 30579 raft_consensus.cc:1270] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e [term 1 FOLLOWER]: Refusing update from remote peer 5168bd8e804e45abae06fee65e7c306d: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
15:06:28.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.324205 30697 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115: Bootstrap starting.
15:06:28.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.324712 30803 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a86734036df442d9067f984db9f965e" member_type: VOTER last_known_addr { host: "127.29.207.254" port: 42611 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:06:28.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.327282 30697 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115: Neither blocks nor log segments found. Creating new log.
15:06:28.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.328727 30697 log.cc:826] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115: Log is configured to *not* fsync() on all Append() calls
15:06:28.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.329056 30789 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e [sys.catalog]: SysCatalogTable state changed. Reason: New leader 5168bd8e804e45abae06fee65e7c306d. Latest consensus state: current_term: 1 leader_uuid: "5168bd8e804e45abae06fee65e7c306d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a86734036df442d9067f984db9f965e" member_type: VOTER last_known_addr { host: "127.29.207.254" port: 42611 } } peers { permanent_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115" member_type: VOTER last_known_addr { host: "127.29.207.253" port: 44061 } } peers { permanent_uuid: "5168bd8e804e45abae06fee65e7c306d" member_type: VOTER last_known_addr { host: "127.29.207.252" port: 42085 } } }
15:06:28.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.329214 30789 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e [sys.catalog]: This master's current role is: FOLLOWER
15:06:28.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.343021 30803 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "5168bd8e804e45abae06fee65e7c306d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a86734036df442d9067f984db9f965e" member_type: VOTER last_known_addr { host: "127.29.207.254" port: 42611 } } peers { permanent_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115" member_type: VOTER last_known_addr { host: "127.29.207.253" port: 44061 } } peers { permanent_uuid: "5168bd8e804e45abae06fee65e7c306d" member_type: VOTER last_known_addr { host: "127.29.207.252" port: 42085 } } }
15:06:28.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.343204 30803 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [sys.catalog]: This master's current role is: LEADER
15:06:28.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.343824 30697 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115: No bootstrap required, opened a new log
15:06:28.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.344872 30789 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "5168bd8e804e45abae06fee65e7c306d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a86734036df442d9067f984db9f965e" member_type: VOTER last_known_addr { host: "127.29.207.254" port: 42611 } } peers { permanent_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115" member_type: VOTER last_known_addr { host: "127.29.207.253" port: 44061 } } peers { permanent_uuid: "5168bd8e804e45abae06fee65e7c306d" member_type: VOTER last_known_addr { host: "127.29.207.252" port: 42085 } } }
15:06:28.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.344995 30789 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e [sys.catalog]: This master's current role is: FOLLOWER
15:06:28.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.345572 30820 mvcc.cc:204] Tried to move back new op lower bound from 7113500214565163008 to 7113500214532239360. Current Snapshot: MvccSnapshot[applied={T|T < 7113500214565163008}]
15:06:28.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.346176 30819 mvcc.cc:204] Tried to move back new op lower bound from 7113500214565163008 to 7113500214532239360. Current Snapshot: MvccSnapshot[applied={T|T < 7113500214565163008}]
15:06:28.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.349619 30697 raft_consensus.cc:357] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a86734036df442d9067f984db9f965e" member_type: VOTER last_known_addr { host: "127.29.207.254" port: 42611 } } peers { permanent_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115" member_type: VOTER last_known_addr { host: "127.29.207.253" port: 44061 } } peers { permanent_uuid: "5168bd8e804e45abae06fee65e7c306d" member_type: VOTER last_known_addr { host: "127.29.207.252" port: 42085 } }
15:06:28.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.350112 30697 raft_consensus.cc:383] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:06:28.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.350327 30697 raft_consensus.cc:738] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 80c9fc0ab13d44ffa2a33e3b8c32f115, State: Initialized, Role: FOLLOWER
15:06:28.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.351199 30697 consensus_queue.cc:260] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a86734036df442d9067f984db9f965e" member_type: VOTER last_known_addr { host: "127.29.207.254" port: 42611 } } peers { permanent_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115" member_type: VOTER last_known_addr { host: "127.29.207.253" port: 44061 } } peers { permanent_uuid: "5168bd8e804e45abae06fee65e7c306d" member_type: VOTER last_known_addr { host: "127.29.207.252" port: 42085 } }
15:06:28.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.353313 30818 catalog_manager.cc:1348] Generated new cluster ID: 3fab646ada5f4cf2b1a6315d5ebfd421
15:06:28.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.353392 30818 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
15:06:28.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.354313 30661 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5168bd8e804e45abae06fee65e7c306d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115" is_pre_election: true
15:06:28.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.354830 30661 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5168bd8e804e45abae06fee65e7c306d in term 0.
15:06:28.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.355468 30697 sys_catalog.cc:564] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115 [sys.catalog]: configured and running, proceeding with master startup.
15:06:28.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.358752 30659 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115 [term 0 FOLLOWER]: Advancing to term 1
15:06:28.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.360065 30659 raft_consensus.cc:1270] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115 [term 1 FOLLOWER]: Refusing update from remote peer 5168bd8e804e45abae06fee65e7c306d: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
15:06:28.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.360634 30660 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5168bd8e804e45abae06fee65e7c306d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115"
15:06:28.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.360708 30816 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [LEADER]: Connected to new peer: Peer: permanent_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115" member_type: VOTER last_known_addr { host: "127.29.207.253" port: 44061 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:06:28.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.361543 30660 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5168bd8e804e45abae06fee65e7c306d in term 1.
15:06:28.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.368490 30836 catalog_manager.cc:1559] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:06:28.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.368587 30836 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:06:28.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.368708 30838 sys_catalog.cc:455] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "5168bd8e804e45abae06fee65e7c306d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a86734036df442d9067f984db9f965e" member_type: VOTER last_known_addr { host: "127.29.207.254" port: 42611 } } peers { permanent_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115" member_type: VOTER last_known_addr { host: "127.29.207.253" port: 44061 } } peers { permanent_uuid: "5168bd8e804e45abae06fee65e7c306d" member_type: VOTER last_known_addr { host: "127.29.207.252" port: 42085 } } }
15:06:28.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.368814 30838 sys_catalog.cc:458] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115 [sys.catalog]: This master's current role is: FOLLOWER
15:06:28.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.368779 30825 sys_catalog.cc:455] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 5168bd8e804e45abae06fee65e7c306d. Latest consensus state: current_term: 1 leader_uuid: "5168bd8e804e45abae06fee65e7c306d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a86734036df442d9067f984db9f965e" member_type: VOTER last_known_addr { host: "127.29.207.254" port: 42611 } } peers { permanent_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115" member_type: VOTER last_known_addr { host: "127.29.207.253" port: 44061 } } peers { permanent_uuid: "5168bd8e804e45abae06fee65e7c306d" member_type: VOTER last_known_addr { host: "127.29.207.252" port: 42085 } } }
15:06:28.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.368858 30825 sys_catalog.cc:458] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115 [sys.catalog]: This master's current role is: FOLLOWER
15:06:28.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.368891 30838 sys_catalog.cc:455] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "5168bd8e804e45abae06fee65e7c306d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a86734036df442d9067f984db9f965e" member_type: VOTER last_known_addr { host: "127.29.207.254" port: 42611 } } peers { permanent_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115" member_type: VOTER last_known_addr { host: "127.29.207.253" port: 44061 } } peers { permanent_uuid: "5168bd8e804e45abae06fee65e7c306d" member_type: VOTER last_known_addr { host: "127.29.207.252" port: 42085 } } }
15:06:28.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.368949 30838 sys_catalog.cc:458] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115 [sys.catalog]: This master's current role is: FOLLOWER
15:06:28.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.372663 30818 catalog_manager.cc:1371] Generated new certificate authority record
15:06:28.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.374020 30818 catalog_manager.cc:1505] Loading token signing keys...
15:06:28.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.380331 30839 mvcc.cc:204] Tried to move back new op lower bound from 7113500214565163008 to 7113500214532239360. Current Snapshot: MvccSnapshot[applied={T|T < 7113500214565163008}]
15:06:28.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.381266 30803 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "5168bd8e804e45abae06fee65e7c306d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a86734036df442d9067f984db9f965e" member_type: VOTER last_known_addr { host: "127.29.207.254" port: 42611 } } peers { permanent_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115" member_type: VOTER last_known_addr { host: "127.29.207.253" port: 44061 } } peers { permanent_uuid: "5168bd8e804e45abae06fee65e7c306d" member_type: VOTER last_known_addr { host: "127.29.207.252" port: 42085 } } }
15:06:28.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.381392 30803 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [sys.catalog]: This master's current role is: LEADER
15:06:28.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.405301 30818 catalog_manager.cc:5899] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d: Generated new TSK 0
15:06:28.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.406440 30818 catalog_manager.cc:1515] Initializing in-progress tserver states...
15:06:28.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.419297 30527 external_mini_cluster.cc:1351] Running /tmp/dist-test-task7b0YEV/build/debug/bin/kudu
15:06:28.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task7b0YEV/build/debug/bin/kudu
15:06:28.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/ts-0/wal
15:06:28.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/ts-0/data
15:06:28.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:06:28.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:06:28.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:06:28.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:28.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:28.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:28.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/ts-0/logs
15:06:28.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/ts-0/data/info.pb
15:06:28.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:28.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:06:28.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:06:28.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:06:28.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:06:28.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:28.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:28.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:28.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:06:28.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:06:28.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.207.193:0
15:06:28.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.207.193
15:06:28.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.207.193
15:06:28.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:06:28.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.207.254:42611,127.29.207.253:44061,127.29.207.252:42085
15:06:28.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.207.212:43549
15:06:28.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:28.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:28.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:28.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
15:06:28.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
15:06:28.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
15:06:28.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.563544 30845 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:06:28.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.563843 30845 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:06:28.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.563935 30845 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:06:28.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.567134 30845 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:06:28.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.567238 30845 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
15:06:28.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.567265 30845 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
15:06:28.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.567286 30845 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
15:06:28.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.567345 30845 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.207.193
15:06:28.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.570889 30845 tablet_server_runner.cc:78] Tablet server non-default flags:
15:06:28.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:28.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.207.212:43549
15:06:28.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:28.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:28.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/ts-0/data
15:06:28.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/ts-0/wal
15:06:28.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:28.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:28.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.207.193:0
15:06:28.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:06:28.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:28.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:28.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/ts-0/data/info.pb
15:06:28.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.207.193
15:06:28.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:06:28.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
15:06:28.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
15:06:28.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
15:06:28.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.207.254:42611,127.29.207.253:44061,127.29.207.252:42085
15:06:28.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:06:28.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.30845
15:06:28.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:28.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:06:28.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:06:28.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:28.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.207.193
15:06:28.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/ts-0/logs
15:06:28.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:28.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:06:28.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:06:28.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:06:28.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:06:28.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:06:28.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
15:06:28.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Jan 2025 14:43:14 UTC on 24a791456cd2
15:06:28.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4141
15:06:28.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.572110 30845 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:06:28.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.573233 30845 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:06:28.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.580992 30853 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:28.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.581200 30845 server_base.cc:1034] running on GCE node
15:06:28.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.581053 30851 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:28.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.581048 30850 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:28.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.582073 30845 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:06:28.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.582677 30845 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:06:28.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.583822 30845 hybrid_clock.cc:648] HybridClock initialized: now 1736694388583799 us; error 43 us; skew 500 ppm
15:06:28.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.585964 30845 webserver.cc:458] Webserver started at http://127.29.207.193:42779/ using document root <none> and password file <none>
15:06:28.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.586591 30845 fs_manager.cc:362] Metadata directory not provided
15:06:28.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.586659 30845 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:06:28.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.586874 30845 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:06:28.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.588573 30845 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/ts-0/data/instance:
15:06:28.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e6001a1b382847169cac813c03722887"
15:06:28.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:06:28 on dist-test-slave-xnn2"
15:06:28.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.589097 30845 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/ts-0/wal/instance:
15:06:28.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e6001a1b382847169cac813c03722887"
15:06:28.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:06:28 on dist-test-slave-xnn2"
15:06:28.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.592594 30845 fs_manager.cc:696] Time spent creating directory manager: real 0.003s user 0.005s sys 0.000s
15:06:28.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.594849 30859 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:06:28.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.595952 30845 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.002s sys 0.001s
15:06:28.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.596060 30845 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/ts-0/data,/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/ts-0/wal
15:06:28.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e6001a1b382847169cac813c03722887"
15:06:28.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:06:28 on dist-test-slave-xnn2"
15:06:28.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.596174 30845 fs_report.cc:389] FS layout report
15:06:28.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:06:28.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/ts-0/wal
15:06:28.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/ts-0/wal
15:06:28.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/ts-0/data/data
15:06:28.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:06:28.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:06:28.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:06:28.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:06:28.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:06:28.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:06:28.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:06:28.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:06:28.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:06:28.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:06:28.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:06:28.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.619967 30845 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:06:28.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.620607 30845 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:06:28.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.620770 30845 kserver.cc:163] Server-wide thread pool size limit: 3276
15:06:28.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.622238 30845 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:06:28.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.622289 30845 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
15:06:28.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.622346 30845 ts_tablet_manager.cc:610] Registered 0 tablets
15:06:28.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.622366 30845 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:06:28.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.632822 30845 rpc_server.cc:307] RPC server started. Bound to: 127.29.207.193:39647
15:06:28.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.632895 30978 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.207.193:39647 every 8 connection(s)
15:06:28.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.633984 30845 server_base.cc:1166] Dumped server information to /tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/ts-0/data/info.pb
15:06:28.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.638204 30983 heartbeater.cc:346] Connected to a master server at 127.29.207.254:42611
15:06:28.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.638239 30527 external_mini_cluster.cc:1413] Started /tmp/dist-test-task7b0YEV/build/debug/bin/kudu as pid 30845
15:06:28.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.638342 30527 external_mini_cluster.cc:1427] Reading /tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/ts-0/wal/instance
15:06:28.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.642278 30527 external_mini_cluster.cc:1351] Running /tmp/dist-test-task7b0YEV/build/debug/bin/kudu
15:06:28.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task7b0YEV/build/debug/bin/kudu
15:06:28.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/ts-1/wal
15:06:28.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/ts-1/data
15:06:28.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:06:28.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:06:28.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:06:28.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:28.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:28.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:28.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/ts-1/logs
15:06:28.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/ts-1/data/info.pb
15:06:28.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:28.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:06:28.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:06:28.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:06:28.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:06:28.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:28.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:28.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:28.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:06:28.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:06:28.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.207.194:0
15:06:28.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.207.194
15:06:28.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.207.194
15:06:28.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:06:28.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.207.254:42611,127.29.207.253:44061,127.29.207.252:42085
15:06:28.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.207.212:43549
15:06:28.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:28.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:28.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:28.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
15:06:28.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
15:06:28.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
15:06:28.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.642721 30983 heartbeater.cc:463] Registering TS with master...
15:06:28.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.643537 30983 heartbeater.cc:510] Master 127.29.207.254:42611 requested a full tablet report, sending...
15:06:28.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.644138 30984 heartbeater.cc:346] Connected to a master server at 127.29.207.253:44061
15:06:28.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.644220 30984 heartbeater.cc:463] Registering TS with master...
15:06:28.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.644546 30982 heartbeater.cc:346] Connected to a master server at 127.29.207.252:42085
15:06:28.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.644601 30984 heartbeater.cc:510] Master 127.29.207.253:44061 requested a full tablet report, sending...
15:06:28.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.644630 30982 heartbeater.cc:463] Registering TS with master...
15:06:28.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.644804 30982 heartbeater.cc:510] Master 127.29.207.252:42085 requested a full tablet report, sending...
15:06:28.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.645236 30568 ts_manager.cc:194] Registered new tserver with Master: e6001a1b382847169cac813c03722887 (127.29.207.193:39647)
15:06:28.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.645567 30651 ts_manager.cc:194] Registered new tserver with Master: e6001a1b382847169cac813c03722887 (127.29.207.193:39647)
15:06:28.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.645763 30733 ts_manager.cc:194] Registered new tserver with Master: e6001a1b382847169cac813c03722887 (127.29.207.193:39647)
15:06:28.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.646971 30733 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.29.207.193:47387
15:06:28.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.750995 30987 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:06:28.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.751209 30987 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:06:28.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.751283 30987 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:06:28.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.754196 30987 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:06:28.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.754299 30987 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
15:06:28.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.754330 30987 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
15:06:28.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.754350 30987 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
15:06:28.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.754400 30987 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.207.194
15:06:28.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.758138 30987 tablet_server_runner.cc:78] Tablet server non-default flags:
15:06:28.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:28.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.207.212:43549
15:06:28.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:28.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:28.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/ts-1/data
15:06:28.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/ts-1/wal
15:06:28.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:28.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:28.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.207.194:0
15:06:28.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:06:28.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:28.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:28.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/ts-1/data/info.pb
15:06:28.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.207.194
15:06:28.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:06:28.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
15:06:28.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
15:06:28.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
15:06:28.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.207.254:42611,127.29.207.253:44061,127.29.207.252:42085
15:06:28.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:06:28.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.30987
15:06:28.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:28.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:06:28.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:06:28.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:28.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.207.194
15:06:28.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/ts-1/logs
15:06:28.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:28.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:06:28.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:06:28.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:06:28.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:06:28.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:06:28.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
15:06:28.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Jan 2025 14:43:14 UTC on 24a791456cd2
15:06:28.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4141
15:06:28.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.759558 30987 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:06:28.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.760665 30987 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:06:28.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.768395 30992 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:28.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.768484 30993 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:28.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.768420 30995 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:28.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.768781 30987 server_base.cc:1034] running on GCE node
15:06:28.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.769155 30987 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:06:28.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.769682 30987 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:06:28.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.770826 30987 hybrid_clock.cc:648] HybridClock initialized: now 1736694388770804 us; error 38 us; skew 500 ppm
15:06:28.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.772661 30987 webserver.cc:458] Webserver started at http://127.29.207.194:34339/ using document root <none> and password file <none>
15:06:28.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.773198 30987 fs_manager.cc:362] Metadata directory not provided
15:06:28.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.773257 30987 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:06:28.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.773444 30987 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:06:28.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.775125 30987 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/ts-1/data/instance:
15:06:28.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aff4ff4877ed42f58b9a3f6430894003"
15:06:28.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:06:28 on dist-test-slave-xnn2"
15:06:28.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.775630 30987 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/ts-1/wal/instance:
15:06:28.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aff4ff4877ed42f58b9a3f6430894003"
15:06:28.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:06:28 on dist-test-slave-xnn2"
15:06:28.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.779227 30987 fs_manager.cc:696] Time spent creating directory manager: real 0.003s user 0.003s sys 0.000s
15:06:28.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.781250 31001 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:06:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.782212 30987 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
15:06:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.782315 30987 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/ts-1/data,/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/ts-1/wal
15:06:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aff4ff4877ed42f58b9a3f6430894003"
15:06:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:06:28 on dist-test-slave-xnn2"
15:06:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.782406 30987 fs_report.cc:389] FS layout report
15:06:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:06:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/ts-1/wal
15:06:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/ts-1/wal
15:06:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/ts-1/data/data
15:06:28.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:06:28.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:06:28.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:06:28.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:06:28.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:06:28.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:06:28.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:06:28.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:06:28.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:06:28.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:06:28.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:06:28.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.798563 30987 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:06:28.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.799186 30987 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:06:28.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.799362 30987 kserver.cc:163] Server-wide thread pool size limit: 3276
15:06:28.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.801035 30987 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:06:28.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.801101 30987 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
15:06:28.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.801149 30987 ts_tablet_manager.cc:610] Registered 0 tablets
15:06:28.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.801188 30987 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:06:28.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.811793 30987 rpc_server.cc:307] RPC server started. Bound to: 127.29.207.194:36203
15:06:28.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.813087 30987 server_base.cc:1166] Dumped server information to /tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/ts-1/data/info.pb
15:06:28.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.814157 31122 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.207.194:36203 every 8 connection(s)
15:06:28.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.814535 31125 heartbeater.cc:346] Connected to a master server at 127.29.207.254:42611
15:06:28.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.814971 31125 heartbeater.cc:463] Registering TS with master...
15:06:28.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.815897 31125 heartbeater.cc:510] Master 127.29.207.254:42611 requested a full tablet report, sending...
15:06:28.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.817512 30568 ts_manager.cc:194] Registered new tserver with Master: aff4ff4877ed42f58b9a3f6430894003 (127.29.207.194:36203)
15:06:28.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.820492 30527 external_mini_cluster.cc:1413] Started /tmp/dist-test-task7b0YEV/build/debug/bin/kudu as pid 30987
15:06:28.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.820596 30527 external_mini_cluster.cc:1427] Reading /tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/ts-1/wal/instance
15:06:28.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.822803 31124 heartbeater.cc:346] Connected to a master server at 127.29.207.252:42085
15:06:28.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.822896 31124 heartbeater.cc:463] Registering TS with master...
15:06:28.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.823092 31124 heartbeater.cc:510] Master 127.29.207.252:42085 requested a full tablet report, sending...
15:06:28.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.823598 31126 heartbeater.cc:346] Connected to a master server at 127.29.207.253:44061
15:06:28.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.823613 30527 external_mini_cluster.cc:1351] Running /tmp/dist-test-task7b0YEV/build/debug/bin/kudu
15:06:28.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task7b0YEV/build/debug/bin/kudu
15:06:28.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/ts-2/wal
15:06:28.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/ts-2/data
15:06:28.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:06:28.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:06:28.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:06:28.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:28.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:28.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:28.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/ts-2/logs
15:06:28.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/ts-2/data/info.pb
15:06:28.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:28.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:06:28.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:06:28.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:06:28.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:06:28.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:28.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:28.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:28.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:06:28.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:06:28.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.207.195:0
15:06:28.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.207.195
15:06:28.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.207.195
15:06:28.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:06:28.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.207.254:42611,127.29.207.253:44061,127.29.207.252:42085
15:06:28.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.207.212:43549
15:06:28.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:28.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:28.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:28.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
15:06:28.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
15:06:28.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
15:06:28.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.823678 31126 heartbeater.cc:463] Registering TS with master...
15:06:28.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.823813 31126 heartbeater.cc:510] Master 127.29.207.253:44061 requested a full tablet report, sending...
15:06:28.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.825284 30651 ts_manager.cc:194] Registered new tserver with Master: aff4ff4877ed42f58b9a3f6430894003 (127.29.207.194:36203)
15:06:28.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.825373 30734 ts_manager.cc:194] Registered new tserver with Master: aff4ff4877ed42f58b9a3f6430894003 (127.29.207.194:36203)
15:06:28.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.826049 30734 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.29.207.194:58947
15:06:28.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.931233 31129 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:06:28.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.931468 31129 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:06:28.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.931555 31129 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:06:28.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.934720 31129 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:06:28.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.934827 31129 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
15:06:28.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.934852 31129 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
15:06:28.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.934872 31129 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
15:06:28.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.934923 31129 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.207.195
15:06:28.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.938701 31129 tablet_server_runner.cc:78] Tablet server non-default flags:
15:06:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.207.212:43549
15:06:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/ts-2/data
15:06:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/ts-2/wal
15:06:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.207.195:0
15:06:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:06:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/ts-2/data/info.pb
15:06:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.207.195
15:06:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:06:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
15:06:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
15:06:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
15:06:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.207.254:42611,127.29.207.253:44061,127.29.207.252:42085
15:06:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:06:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.31129
15:06:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:06:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:06:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.207.195
15:06:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/ts-2/logs
15:06:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:06:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:06:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:06:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:06:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:06:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
15:06:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Jan 2025 14:43:14 UTC on 24a791456cd2
15:06:28.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4141
15:06:28.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.940030 31129 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:06:28.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.941161 31129 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:06:28.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.949043 31135 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:28.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.949107 31129 server_base.cc:1034] running on GCE node
15:06:28.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.949201 31137 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:28.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:28.949188 31134 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:28.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.949776 31129 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:06:28.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.950358 31129 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:06:28.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.951521 31129 hybrid_clock.cc:648] HybridClock initialized: now 1736694388951511 us; error 34 us; skew 500 ppm
15:06:28.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.960428 31129 webserver.cc:458] Webserver started at http://127.29.207.195:39329/ using document root <none> and password file <none>
15:06:28.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.961052 31129 fs_manager.cc:362] Metadata directory not provided
15:06:28.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.961117 31129 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:06:28.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.961318 31129 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:06:28.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.963110 31129 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/ts-2/data/instance:
15:06:28.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "57641dcf709949098d39f0e8b0fbe6a7"
15:06:28.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:06:28 on dist-test-slave-xnn2"
15:06:28.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.963650 31129 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/ts-2/wal/instance:
15:06:28.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "57641dcf709949098d39f0e8b0fbe6a7"
15:06:28.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:06:28 on dist-test-slave-xnn2"
15:06:28.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.967221 31129 fs_manager.cc:696] Time spent creating directory manager: real 0.003s user 0.004s sys 0.000s
15:06:28.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.969389 31143 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:06:28.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.970418 31129 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
15:06:28.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.970556 31129 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/ts-2/data,/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/ts-2/wal
15:06:28.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "57641dcf709949098d39f0e8b0fbe6a7"
15:06:28.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:06:28 on dist-test-slave-xnn2"
15:06:28.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.970641 31129 fs_report.cc:389] FS layout report
15:06:28.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:06:28.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/ts-2/wal
15:06:28.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/ts-2/wal
15:06:28.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/ts-2/data/data
15:06:28.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:06:28.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:06:28.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:06:28.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:06:28.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:06:28.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:06:28.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:06:28.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:06:28.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:06:28.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:06:28.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:06:28.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.981334 31129 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:06:28.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.981859 31129 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:06:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.981989 31129 kserver.cc:163] Server-wide thread pool size limit: 3276
15:06:28.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.983419 31129 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:06:28.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.983470 31129 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
15:06:28.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.983522 31129 ts_tablet_manager.cc:610] Registered 0 tablets
15:06:28.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.983541 31129 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:06:28.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.994161 31129 rpc_server.cc:307] RPC server started. Bound to: 127.29.207.195:40677
15:06:28.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.995301 31129 server_base.cc:1166] Dumped server information to /tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/ts-2/data/info.pb
15:06:28.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.994203 31264 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.207.195:40677 every 8 connection(s)
15:06:28.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.997967 31268 heartbeater.cc:346] Connected to a master server at 127.29.207.254:42611
15:06:28.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.998217 31268 heartbeater.cc:463] Registering TS with master...
15:06:28.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:28.998883 31268 heartbeater.cc:510] Master 127.29.207.254:42611 requested a full tablet report, sending...
15:06:29.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.000046 30568 ts_manager.cc:194] Registered new tserver with Master: 57641dcf709949098d39f0e8b0fbe6a7 (127.29.207.195:40677)
15:06:29.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.000147 30527 external_mini_cluster.cc:1413] Started /tmp/dist-test-task7b0YEV/build/debug/bin/kudu as pid 31129
15:06:29.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.000244 30527 external_mini_cluster.cc:1427] Reading /tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/ts-2/wal/instance
15:06:29.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.001649 31267 heartbeater.cc:346] Connected to a master server at 127.29.207.252:42085
15:06:29.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.001833 31267 heartbeater.cc:463] Registering TS with master...
15:06:29.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.002132 31267 heartbeater.cc:510] Master 127.29.207.252:42085 requested a full tablet report, sending...
15:06:29.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.002808 30733 ts_manager.cc:194] Registered new tserver with Master: 57641dcf709949098d39f0e8b0fbe6a7 (127.29.207.195:40677)
15:06:29.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.003543 30733 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.29.207.195:34843
15:06:29.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.004806 31269 heartbeater.cc:346] Connected to a master server at 127.29.207.253:44061
15:06:29.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.004876 31269 heartbeater.cc:463] Registering TS with master...
15:06:29.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.005019 31269 heartbeater.cc:510] Master 127.29.207.253:44061 requested a full tablet report, sending...
15:06:29.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.005844 30651 ts_manager.cc:194] Registered new tserver with Master: 57641dcf709949098d39f0e8b0fbe6a7 (127.29.207.195:40677)
15:06:29.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.009151 30527 external_mini_cluster.cc:934] 3 TS(s) registered with all masters
15:06:29.009 [DEBUG - main] (MiniKuduCluster.java:195) Response:
15:06:29.010 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
15:06:29.010 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.29.207.254"
port: 42611
}
bound_http_address {
host: "127.29.207.254"
port: 40757
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.29.207.253"
port: 44061
}
bound_http_address {
host: "127.29.207.253"
port: 34259
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.29.207.252"
port: 42085
}
bound_http_address {
host: "127.29.207.252"
port: 37191
}
}
}
15:06:29.010 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
15:06:29.011 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.29.207.193"
port: 39647
}
bound_http_address {
host: "127.29.207.193"
port: 42779
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.29.207.194"
port: 36203
}
bound_http_address {
host: "127.29.207.194"
port: 34339
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.29.207.195"
port: 40677
}
bound_http_address {
host: "127.29.207.195"
port: 39329
}
}
}
15:06:29.011 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
15:06:29.018 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
15:06:29.020 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.29.207.254:42611(127.29.207.254:42611)] connecting to peer
15:06:29.020 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.29.207.252:42085(127.29.207.252:42085)] connecting to peer
15:06:29.021 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.29.207.253:44061(127.29.207.253:44061)] connecting to peer
15:06:29.021 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.29.207.254:42611(127.29.207.254:42611)] Successfully connected to peer
15:06:29.023 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.29.207.252:42085(127.29.207.252:42085)] Successfully connected to peer
15:06:29.024 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.29.207.254
15:06:29.025 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.29.207.252
15:06:29.027 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.29.207.253:44061(127.29.207.253:44061)] Successfully connected to peer
15:06:29.028 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.29.207.253
15:06:29.033 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x2d256eac, L:/127.0.0.1:44258 - R:/127.29.207.252:42085] using SASL/PLAIN
15:06:29.035 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
15:06:29.036 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=16), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.254:42611, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.252:42085, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.253:44061, [15ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.252:42085: OK, deferred=null) after lookup
15:06:29.042 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x5ca007b5, L:/127.0.0.1:50684 - R:/127.29.207.253:44061] using SASL/PLAIN
15:06:29.043 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x3cae0cf2, L:/127.0.0.1:54038 - R:/127.29.207.254:42611] using SASL/PLAIN
15:06:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.042717 30733 catalog_manager.cc:1909] Servicing CreateTable request from {username='slave'} at 127.0.0.1:44258:
15:06:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_manager_fallback_rolling"
15:06:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
15:06:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:06:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
15:06:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
15:06:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
15:06:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
15:06:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
15:06:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
15:06:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
15:06:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:06:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:06:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column1_i"
15:06:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
15:06:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
15:06:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
15:06:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
15:06:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
15:06:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
15:06:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:06:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:06:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column2_i"
15:06:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
15:06:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
15:06:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
15:06:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
15:06:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
15:06:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
15:06:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:06:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:06:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column3_s"
15:06:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
15:06:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
15:06:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
15:06:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: DICT_ENCODING
15:06:29.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: LZ4
15:06:29.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 4096
15:06:29.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
15:06:29.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
15:06:29.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:06:29.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:06:29.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column4_b"
15:06:29.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: BOOL
15:06:29.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
15:06:29.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
15:06:29.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
15:06:29.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
15:06:29.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
15:06:29.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:06:29.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:06:29.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
15:06:29.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) hash_schema {
15:06:29.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:06:29.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
15:06:29.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:06:29.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_buckets: 2
15:06:29.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) seed: 0
15:06:29.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:06:29.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:06:29.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:29.044435 30733 catalog_manager.cc:6885] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table txn_manager_fallback_rolling in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
15:06:29.058 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table 5661f08abeeb4d1483bc1ec2b552a012
15:06:29.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.067538 31193 tablet_service.cc:1467] Processing CreateTablet for tablet c0712e7c0e614a8a9c9ebb68d8bc2e71 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=5661f08abeeb4d1483bc1ec2b552a012]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
15:06:29.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.067539 31194 tablet_service.cc:1467] Processing CreateTablet for tablet 68b0a288f3184acfb28ff0691b224244 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=5661f08abeeb4d1483bc1ec2b552a012]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
15:06:29.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.068434 31193 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c0712e7c0e614a8a9c9ebb68d8bc2e71. 1 dirs total, 0 dirs full, 0 dirs failed
15:06:29.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.068666 31194 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 68b0a288f3184acfb28ff0691b224244. 1 dirs total, 0 dirs full, 0 dirs failed
15:06:29.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.073361 31288 tablet_bootstrap.cc:492] T 68b0a288f3184acfb28ff0691b224244 P 57641dcf709949098d39f0e8b0fbe6a7: Bootstrap starting.
15:06:29.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.075342 31288 tablet_bootstrap.cc:654] T 68b0a288f3184acfb28ff0691b224244 P 57641dcf709949098d39f0e8b0fbe6a7: Neither blocks nor log segments found. Creating new log.
15:06:29.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.075953 31288 log.cc:826] T 68b0a288f3184acfb28ff0691b224244 P 57641dcf709949098d39f0e8b0fbe6a7: Log is configured to *not* fsync() on all Append() calls
15:06:29.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.077404 31288 tablet_bootstrap.cc:492] T 68b0a288f3184acfb28ff0691b224244 P 57641dcf709949098d39f0e8b0fbe6a7: No bootstrap required, opened a new log
15:06:29.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.077545 31288 ts_tablet_manager.cc:1397] T 68b0a288f3184acfb28ff0691b224244 P 57641dcf709949098d39f0e8b0fbe6a7: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
15:06:29.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.080531 31288 raft_consensus.cc:357] T 68b0a288f3184acfb28ff0691b224244 P 57641dcf709949098d39f0e8b0fbe6a7 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6001a1b382847169cac813c03722887" member_type: VOTER last_known_addr { host: "127.29.207.193" port: 39647 } } peers { permanent_uuid: "57641dcf709949098d39f0e8b0fbe6a7" member_type: VOTER last_known_addr { host: "127.29.207.195" port: 40677 } } peers { permanent_uuid: "aff4ff4877ed42f58b9a3f6430894003" member_type: VOTER last_known_addr { host: "127.29.207.194" port: 36203 } }
15:06:29.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.080731 31288 raft_consensus.cc:383] T 68b0a288f3184acfb28ff0691b224244 P 57641dcf709949098d39f0e8b0fbe6a7 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:06:29.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.080780 31288 raft_consensus.cc:738] T 68b0a288f3184acfb28ff0691b224244 P 57641dcf709949098d39f0e8b0fbe6a7 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 57641dcf709949098d39f0e8b0fbe6a7, State: Initialized, Role: FOLLOWER
15:06:29.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.081243 31288 consensus_queue.cc:260] T 68b0a288f3184acfb28ff0691b224244 P 57641dcf709949098d39f0e8b0fbe6a7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6001a1b382847169cac813c03722887" member_type: VOTER last_known_addr { host: "127.29.207.193" port: 39647 } } peers { permanent_uuid: "57641dcf709949098d39f0e8b0fbe6a7" member_type: VOTER last_known_addr { host: "127.29.207.195" port: 40677 } } peers { permanent_uuid: "aff4ff4877ed42f58b9a3f6430894003" member_type: VOTER last_known_addr { host: "127.29.207.194" port: 36203 } }
15:06:29.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.081964 31267 heartbeater.cc:502] Master 127.29.207.252:42085 was elected leader, sending a full tablet report...
15:06:29.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.082167 31288 ts_tablet_manager.cc:1428] T 68b0a288f3184acfb28ff0691b224244 P 57641dcf709949098d39f0e8b0fbe6a7: Time spent starting tablet: real 0.005s user 0.005s sys 0.000s
15:06:29.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.083019 31288 tablet_bootstrap.cc:492] T c0712e7c0e614a8a9c9ebb68d8bc2e71 P 57641dcf709949098d39f0e8b0fbe6a7: Bootstrap starting.
15:06:29.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.083997 31288 tablet_bootstrap.cc:654] T c0712e7c0e614a8a9c9ebb68d8bc2e71 P 57641dcf709949098d39f0e8b0fbe6a7: Neither blocks nor log segments found. Creating new log.
15:06:29.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.085183 31288 tablet_bootstrap.cc:492] T c0712e7c0e614a8a9c9ebb68d8bc2e71 P 57641dcf709949098d39f0e8b0fbe6a7: No bootstrap required, opened a new log
15:06:29.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.085261 31288 ts_tablet_manager.cc:1397] T c0712e7c0e614a8a9c9ebb68d8bc2e71 P 57641dcf709949098d39f0e8b0fbe6a7: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
15:06:29.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.085640 31288 raft_consensus.cc:357] T c0712e7c0e614a8a9c9ebb68d8bc2e71 P 57641dcf709949098d39f0e8b0fbe6a7 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6001a1b382847169cac813c03722887" member_type: VOTER last_known_addr { host: "127.29.207.193" port: 39647 } } peers { permanent_uuid: "aff4ff4877ed42f58b9a3f6430894003" member_type: VOTER last_known_addr { host: "127.29.207.194" port: 36203 } } peers { permanent_uuid: "57641dcf709949098d39f0e8b0fbe6a7" member_type: VOTER last_known_addr { host: "127.29.207.195" port: 40677 } }
15:06:29.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.085728 31288 raft_consensus.cc:383] T c0712e7c0e614a8a9c9ebb68d8bc2e71 P 57641dcf709949098d39f0e8b0fbe6a7 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:06:29.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.085749 31288 raft_consensus.cc:738] T c0712e7c0e614a8a9c9ebb68d8bc2e71 P 57641dcf709949098d39f0e8b0fbe6a7 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 57641dcf709949098d39f0e8b0fbe6a7, State: Initialized, Role: FOLLOWER
15:06:29.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.085855 31288 consensus_queue.cc:260] T c0712e7c0e614a8a9c9ebb68d8bc2e71 P 57641dcf709949098d39f0e8b0fbe6a7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6001a1b382847169cac813c03722887" member_type: VOTER last_known_addr { host: "127.29.207.193" port: 39647 } } peers { permanent_uuid: "aff4ff4877ed42f58b9a3f6430894003" member_type: VOTER last_known_addr { host: "127.29.207.194" port: 36203 } } peers { permanent_uuid: "57641dcf709949098d39f0e8b0fbe6a7" member_type: VOTER last_known_addr { host: "127.29.207.195" port: 40677 } }
15:06:29.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.086108 31288 ts_tablet_manager.cc:1428] T c0712e7c0e614a8a9c9ebb68d8bc2e71 P 57641dcf709949098d39f0e8b0fbe6a7: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
15:06:29.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.097949 30908 tablet_service.cc:1467] Processing CreateTablet for tablet c0712e7c0e614a8a9c9ebb68d8bc2e71 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=5661f08abeeb4d1483bc1ec2b552a012]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
15:06:29.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.097939 30909 tablet_service.cc:1467] Processing CreateTablet for tablet 68b0a288f3184acfb28ff0691b224244 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=5661f08abeeb4d1483bc1ec2b552a012]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
15:06:29.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.099001 30908 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c0712e7c0e614a8a9c9ebb68d8bc2e71. 1 dirs total, 0 dirs full, 0 dirs failed
15:06:29.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.099391 30909 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 68b0a288f3184acfb28ff0691b224244. 1 dirs total, 0 dirs full, 0 dirs failed
15:06:29.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.102841 31051 tablet_service.cc:1467] Processing CreateTablet for tablet 68b0a288f3184acfb28ff0691b224244 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=5661f08abeeb4d1483bc1ec2b552a012]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
15:06:29.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.103948 31051 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 68b0a288f3184acfb28ff0691b224244. 1 dirs total, 0 dirs full, 0 dirs failed
15:06:29.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.104727 31050 tablet_service.cc:1467] Processing CreateTablet for tablet c0712e7c0e614a8a9c9ebb68d8bc2e71 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=5661f08abeeb4d1483bc1ec2b552a012]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
15:06:29.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.105096 31050 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c0712e7c0e614a8a9c9ebb68d8bc2e71. 1 dirs total, 0 dirs full, 0 dirs failed
15:06:29.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.106283 31293 tablet_bootstrap.cc:492] T 68b0a288f3184acfb28ff0691b224244 P e6001a1b382847169cac813c03722887: Bootstrap starting.
15:06:29.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.108698 31293 tablet_bootstrap.cc:654] T 68b0a288f3184acfb28ff0691b224244 P e6001a1b382847169cac813c03722887: Neither blocks nor log segments found. Creating new log.
15:06:29.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.109458 31293 log.cc:826] T 68b0a288f3184acfb28ff0691b224244 P e6001a1b382847169cac813c03722887: Log is configured to *not* fsync() on all Append() calls
15:06:29.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.109731 31294 tablet_bootstrap.cc:492] T c0712e7c0e614a8a9c9ebb68d8bc2e71 P aff4ff4877ed42f58b9a3f6430894003: Bootstrap starting.
15:06:29.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.111703 31294 tablet_bootstrap.cc:654] T c0712e7c0e614a8a9c9ebb68d8bc2e71 P aff4ff4877ed42f58b9a3f6430894003: Neither blocks nor log segments found. Creating new log.
15:06:29.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.111938 31293 tablet_bootstrap.cc:492] T 68b0a288f3184acfb28ff0691b224244 P e6001a1b382847169cac813c03722887: No bootstrap required, opened a new log
15:06:29.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.112080 31293 ts_tablet_manager.cc:1397] T 68b0a288f3184acfb28ff0691b224244 P e6001a1b382847169cac813c03722887: Time spent bootstrapping tablet: real 0.006s user 0.004s sys 0.000s
15:06:29.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.112397 31294 log.cc:826] T c0712e7c0e614a8a9c9ebb68d8bc2e71 P aff4ff4877ed42f58b9a3f6430894003: Log is configured to *not* fsync() on all Append() calls
15:06:29.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.113703 31294 tablet_bootstrap.cc:492] T c0712e7c0e614a8a9c9ebb68d8bc2e71 P aff4ff4877ed42f58b9a3f6430894003: No bootstrap required, opened a new log
15:06:29.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.113858 31294 ts_tablet_manager.cc:1397] T c0712e7c0e614a8a9c9ebb68d8bc2e71 P aff4ff4877ed42f58b9a3f6430894003: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
15:06:29.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.115319 31293 raft_consensus.cc:357] T 68b0a288f3184acfb28ff0691b224244 P e6001a1b382847169cac813c03722887 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6001a1b382847169cac813c03722887" member_type: VOTER last_known_addr { host: "127.29.207.193" port: 39647 } } peers { permanent_uuid: "57641dcf709949098d39f0e8b0fbe6a7" member_type: VOTER last_known_addr { host: "127.29.207.195" port: 40677 } } peers { permanent_uuid: "aff4ff4877ed42f58b9a3f6430894003" member_type: VOTER last_known_addr { host: "127.29.207.194" port: 36203 } }
15:06:29.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.115569 31293 raft_consensus.cc:383] T 68b0a288f3184acfb28ff0691b224244 P e6001a1b382847169cac813c03722887 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:06:29.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.115628 31293 raft_consensus.cc:738] T 68b0a288f3184acfb28ff0691b224244 P e6001a1b382847169cac813c03722887 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e6001a1b382847169cac813c03722887, State: Initialized, Role: FOLLOWER
15:06:29.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.116199 31293 consensus_queue.cc:260] T 68b0a288f3184acfb28ff0691b224244 P e6001a1b382847169cac813c03722887 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6001a1b382847169cac813c03722887" member_type: VOTER last_known_addr { host: "127.29.207.193" port: 39647 } } peers { permanent_uuid: "57641dcf709949098d39f0e8b0fbe6a7" member_type: VOTER last_known_addr { host: "127.29.207.195" port: 40677 } } peers { permanent_uuid: "aff4ff4877ed42f58b9a3f6430894003" member_type: VOTER last_known_addr { host: "127.29.207.194" port: 36203 } }
15:06:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.117125 30982 heartbeater.cc:502] Master 127.29.207.252:42085 was elected leader, sending a full tablet report...
15:06:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.117323 31293 ts_tablet_manager.cc:1428] T 68b0a288f3184acfb28ff0691b224244 P e6001a1b382847169cac813c03722887: Time spent starting tablet: real 0.005s user 0.006s sys 0.000s
15:06:29.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.117622 31294 raft_consensus.cc:357] T c0712e7c0e614a8a9c9ebb68d8bc2e71 P aff4ff4877ed42f58b9a3f6430894003 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6001a1b382847169cac813c03722887" member_type: VOTER last_known_addr { host: "127.29.207.193" port: 39647 } } peers { permanent_uuid: "aff4ff4877ed42f58b9a3f6430894003" member_type: VOTER last_known_addr { host: "127.29.207.194" port: 36203 } } peers { permanent_uuid: "57641dcf709949098d39f0e8b0fbe6a7" member_type: VOTER last_known_addr { host: "127.29.207.195" port: 40677 } }
15:06:29.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.117806 31294 raft_consensus.cc:383] T c0712e7c0e614a8a9c9ebb68d8bc2e71 P aff4ff4877ed42f58b9a3f6430894003 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:06:29.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.117856 31294 raft_consensus.cc:738] T c0712e7c0e614a8a9c9ebb68d8bc2e71 P aff4ff4877ed42f58b9a3f6430894003 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: aff4ff4877ed42f58b9a3f6430894003, State: Initialized, Role: FOLLOWER
15:06:29.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.118284 31294 consensus_queue.cc:260] T c0712e7c0e614a8a9c9ebb68d8bc2e71 P aff4ff4877ed42f58b9a3f6430894003 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6001a1b382847169cac813c03722887" member_type: VOTER last_known_addr { host: "127.29.207.193" port: 39647 } } peers { permanent_uuid: "aff4ff4877ed42f58b9a3f6430894003" member_type: VOTER last_known_addr { host: "127.29.207.194" port: 36203 } } peers { permanent_uuid: "57641dcf709949098d39f0e8b0fbe6a7" member_type: VOTER last_known_addr { host: "127.29.207.195" port: 40677 } }
15:06:29.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.119149 31294 ts_tablet_manager.cc:1428] T c0712e7c0e614a8a9c9ebb68d8bc2e71 P aff4ff4877ed42f58b9a3f6430894003: Time spent starting tablet: real 0.005s user 0.003s sys 0.003s
15:06:29.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.119621 31124 heartbeater.cc:502] Master 127.29.207.252:42085 was elected leader, sending a full tablet report...
15:06:29.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.119755 31293 tablet_bootstrap.cc:492] T c0712e7c0e614a8a9c9ebb68d8bc2e71 P e6001a1b382847169cac813c03722887: Bootstrap starting.
15:06:29.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.121162 31293 tablet_bootstrap.cc:654] T c0712e7c0e614a8a9c9ebb68d8bc2e71 P e6001a1b382847169cac813c03722887: Neither blocks nor log segments found. Creating new log.
15:06:29.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.121187 31294 tablet_bootstrap.cc:492] T 68b0a288f3184acfb28ff0691b224244 P aff4ff4877ed42f58b9a3f6430894003: Bootstrap starting.
15:06:29.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.122141 31294 tablet_bootstrap.cc:654] T 68b0a288f3184acfb28ff0691b224244 P aff4ff4877ed42f58b9a3f6430894003: Neither blocks nor log segments found. Creating new log.
15:06:29.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.123940 31294 tablet_bootstrap.cc:492] T 68b0a288f3184acfb28ff0691b224244 P aff4ff4877ed42f58b9a3f6430894003: No bootstrap required, opened a new log
15:06:29.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.124140 31294 ts_tablet_manager.cc:1397] T 68b0a288f3184acfb28ff0691b224244 P aff4ff4877ed42f58b9a3f6430894003: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
15:06:29.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.124698 31294 raft_consensus.cc:357] T 68b0a288f3184acfb28ff0691b224244 P aff4ff4877ed42f58b9a3f6430894003 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6001a1b382847169cac813c03722887" member_type: VOTER last_known_addr { host: "127.29.207.193" port: 39647 } } peers { permanent_uuid: "57641dcf709949098d39f0e8b0fbe6a7" member_type: VOTER last_known_addr { host: "127.29.207.195" port: 40677 } } peers { permanent_uuid: "aff4ff4877ed42f58b9a3f6430894003" member_type: VOTER last_known_addr { host: "127.29.207.194" port: 36203 } }
15:06:29.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.124909 31294 raft_consensus.cc:383] T 68b0a288f3184acfb28ff0691b224244 P aff4ff4877ed42f58b9a3f6430894003 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:06:29.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.124965 31294 raft_consensus.cc:738] T 68b0a288f3184acfb28ff0691b224244 P aff4ff4877ed42f58b9a3f6430894003 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: aff4ff4877ed42f58b9a3f6430894003, State: Initialized, Role: FOLLOWER
15:06:29.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.125118 31294 consensus_queue.cc:260] T 68b0a288f3184acfb28ff0691b224244 P aff4ff4877ed42f58b9a3f6430894003 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6001a1b382847169cac813c03722887" member_type: VOTER last_known_addr { host: "127.29.207.193" port: 39647 } } peers { permanent_uuid: "57641dcf709949098d39f0e8b0fbe6a7" member_type: VOTER last_known_addr { host: "127.29.207.195" port: 40677 } } peers { permanent_uuid: "aff4ff4877ed42f58b9a3f6430894003" member_type: VOTER last_known_addr { host: "127.29.207.194" port: 36203 } }
15:06:29.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.125504 31294 ts_tablet_manager.cc:1428] T 68b0a288f3184acfb28ff0691b224244 P aff4ff4877ed42f58b9a3f6430894003: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
15:06:29.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.128643 31293 tablet_bootstrap.cc:492] T c0712e7c0e614a8a9c9ebb68d8bc2e71 P e6001a1b382847169cac813c03722887: No bootstrap required, opened a new log
15:06:29.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.128728 31293 ts_tablet_manager.cc:1397] T c0712e7c0e614a8a9c9ebb68d8bc2e71 P e6001a1b382847169cac813c03722887: Time spent bootstrapping tablet: real 0.009s user 0.002s sys 0.000s
15:06:29.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.129076 31293 raft_consensus.cc:357] T c0712e7c0e614a8a9c9ebb68d8bc2e71 P e6001a1b382847169cac813c03722887 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6001a1b382847169cac813c03722887" member_type: VOTER last_known_addr { host: "127.29.207.193" port: 39647 } } peers { permanent_uuid: "aff4ff4877ed42f58b9a3f6430894003" member_type: VOTER last_known_addr { host: "127.29.207.194" port: 36203 } } peers { permanent_uuid: "57641dcf709949098d39f0e8b0fbe6a7" member_type: VOTER last_known_addr { host: "127.29.207.195" port: 40677 } }
15:06:29.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.129161 31293 raft_consensus.cc:383] T c0712e7c0e614a8a9c9ebb68d8bc2e71 P e6001a1b382847169cac813c03722887 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:06:29.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.129181 31293 raft_consensus.cc:738] T c0712e7c0e614a8a9c9ebb68d8bc2e71 P e6001a1b382847169cac813c03722887 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e6001a1b382847169cac813c03722887, State: Initialized, Role: FOLLOWER
15:06:29.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.129271 31293 consensus_queue.cc:260] T c0712e7c0e614a8a9c9ebb68d8bc2e71 P e6001a1b382847169cac813c03722887 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6001a1b382847169cac813c03722887" member_type: VOTER last_known_addr { host: "127.29.207.193" port: 39647 } } peers { permanent_uuid: "aff4ff4877ed42f58b9a3f6430894003" member_type: VOTER last_known_addr { host: "127.29.207.194" port: 36203 } } peers { permanent_uuid: "57641dcf709949098d39f0e8b0fbe6a7" member_type: VOTER last_known_addr { host: "127.29.207.195" port: 40677 } }
15:06:29.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.129472 31293 ts_tablet_manager.cc:1428] T c0712e7c0e614a8a9c9ebb68d8bc2e71 P e6001a1b382847169cac813c03722887: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
15:06:29.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:29.136618 30985 tablet.cc:2367] T 68b0a288f3184acfb28ff0691b224244 P e6001a1b382847169cac813c03722887: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:06:29.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.140084 31297 raft_consensus.cc:491] T c0712e7c0e614a8a9c9ebb68d8bc2e71 P e6001a1b382847169cac813c03722887 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:06:29.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.140281 31297 raft_consensus.cc:513] T c0712e7c0e614a8a9c9ebb68d8bc2e71 P e6001a1b382847169cac813c03722887 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6001a1b382847169cac813c03722887" member_type: VOTER last_known_addr { host: "127.29.207.193" port: 39647 } } peers { permanent_uuid: "aff4ff4877ed42f58b9a3f6430894003" member_type: VOTER last_known_addr { host: "127.29.207.194" port: 36203 } } peers { permanent_uuid: "57641dcf709949098d39f0e8b0fbe6a7" member_type: VOTER last_known_addr { host: "127.29.207.195" port: 40677 } }
15:06:29.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.141535 31297 leader_election.cc:290] T c0712e7c0e614a8a9c9ebb68d8bc2e71 P e6001a1b382847169cac813c03722887 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers aff4ff4877ed42f58b9a3f6430894003 (127.29.207.194:36203), 57641dcf709949098d39f0e8b0fbe6a7 (127.29.207.195:40677)
15:06:29.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.145961 31071 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c0712e7c0e614a8a9c9ebb68d8bc2e71" candidate_uuid: "e6001a1b382847169cac813c03722887" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "aff4ff4877ed42f58b9a3f6430894003" is_pre_election: true
15:06:29.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.146049 31290 raft_consensus.cc:491] T 68b0a288f3184acfb28ff0691b224244 P 57641dcf709949098d39f0e8b0fbe6a7 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:06:29.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.146273 31214 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c0712e7c0e614a8a9c9ebb68d8bc2e71" candidate_uuid: "e6001a1b382847169cac813c03722887" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "57641dcf709949098d39f0e8b0fbe6a7" is_pre_election: true
15:06:29.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.146286 31071 raft_consensus.cc:2463] T c0712e7c0e614a8a9c9ebb68d8bc2e71 P aff4ff4877ed42f58b9a3f6430894003 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e6001a1b382847169cac813c03722887 in term 0.
15:06:29.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.146459 31214 raft_consensus.cc:2463] T c0712e7c0e614a8a9c9ebb68d8bc2e71 P 57641dcf709949098d39f0e8b0fbe6a7 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e6001a1b382847169cac813c03722887 in term 0.
15:06:29.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.146243 31290 raft_consensus.cc:513] T 68b0a288f3184acfb28ff0691b224244 P 57641dcf709949098d39f0e8b0fbe6a7 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6001a1b382847169cac813c03722887" member_type: VOTER last_known_addr { host: "127.29.207.193" port: 39647 } } peers { permanent_uuid: "57641dcf709949098d39f0e8b0fbe6a7" member_type: VOTER last_known_addr { host: "127.29.207.195" port: 40677 } } peers { permanent_uuid: "aff4ff4877ed42f58b9a3f6430894003" member_type: VOTER last_known_addr { host: "127.29.207.194" port: 36203 } }
15:06:29.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.146793 30862 leader_election.cc:304] T c0712e7c0e614a8a9c9ebb68d8bc2e71 P e6001a1b382847169cac813c03722887 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: aff4ff4877ed42f58b9a3f6430894003, e6001a1b382847169cac813c03722887; no voters:
15:06:29.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.147048 31297 raft_consensus.cc:2798] T c0712e7c0e614a8a9c9ebb68d8bc2e71 P e6001a1b382847169cac813c03722887 [term 0 FOLLOWER]: Leader pre-election won for term 1
15:06:29.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.147132 31297 raft_consensus.cc:491] T c0712e7c0e614a8a9c9ebb68d8bc2e71 P e6001a1b382847169cac813c03722887 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
15:06:29.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.147176 31297 raft_consensus.cc:3054] T c0712e7c0e614a8a9c9ebb68d8bc2e71 P e6001a1b382847169cac813c03722887 [term 0 FOLLOWER]: Advancing to term 1
15:06:29.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.147609 31290 leader_election.cc:290] T 68b0a288f3184acfb28ff0691b224244 P 57641dcf709949098d39f0e8b0fbe6a7 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e6001a1b382847169cac813c03722887 (127.29.207.193:39647), aff4ff4877ed42f58b9a3f6430894003 (127.29.207.194:36203)
15:06:29.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.148249 31297 raft_consensus.cc:513] T c0712e7c0e614a8a9c9ebb68d8bc2e71 P e6001a1b382847169cac813c03722887 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6001a1b382847169cac813c03722887" member_type: VOTER last_known_addr { host: "127.29.207.193" port: 39647 } } peers { permanent_uuid: "aff4ff4877ed42f58b9a3f6430894003" member_type: VOTER last_known_addr { host: "127.29.207.194" port: 36203 } } peers { permanent_uuid: "57641dcf709949098d39f0e8b0fbe6a7" member_type: VOTER last_known_addr { host: "127.29.207.195" port: 40677 } }
15:06:29.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.148599 31297 leader_election.cc:290] T c0712e7c0e614a8a9c9ebb68d8bc2e71 P e6001a1b382847169cac813c03722887 [CANDIDATE]: Term 1 election: Requested vote from peers aff4ff4877ed42f58b9a3f6430894003 (127.29.207.194:36203), 57641dcf709949098d39f0e8b0fbe6a7 (127.29.207.195:40677)
15:06:29.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.149123 31071 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c0712e7c0e614a8a9c9ebb68d8bc2e71" candidate_uuid: "e6001a1b382847169cac813c03722887" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "aff4ff4877ed42f58b9a3f6430894003"
15:06:29.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.149245 31071 raft_consensus.cc:3054] T c0712e7c0e614a8a9c9ebb68d8bc2e71 P aff4ff4877ed42f58b9a3f6430894003 [term 0 FOLLOWER]: Advancing to term 1
15:06:29.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.149647 31214 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c0712e7c0e614a8a9c9ebb68d8bc2e71" candidate_uuid: "e6001a1b382847169cac813c03722887" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "57641dcf709949098d39f0e8b0fbe6a7"
15:06:29.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.149747 31214 raft_consensus.cc:3054] T c0712e7c0e614a8a9c9ebb68d8bc2e71 P 57641dcf709949098d39f0e8b0fbe6a7 [term 0 FOLLOWER]: Advancing to term 1
15:06:29.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.150308 31071 raft_consensus.cc:2463] T c0712e7c0e614a8a9c9ebb68d8bc2e71 P aff4ff4877ed42f58b9a3f6430894003 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e6001a1b382847169cac813c03722887 in term 1.
15:06:29.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.150661 30862 leader_election.cc:304] T c0712e7c0e614a8a9c9ebb68d8bc2e71 P e6001a1b382847169cac813c03722887 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: aff4ff4877ed42f58b9a3f6430894003, e6001a1b382847169cac813c03722887; no voters:
15:06:29.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.150803 31214 raft_consensus.cc:2463] T c0712e7c0e614a8a9c9ebb68d8bc2e71 P 57641dcf709949098d39f0e8b0fbe6a7 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e6001a1b382847169cac813c03722887 in term 1.
15:06:29.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.150822 31297 raft_consensus.cc:2798] T c0712e7c0e614a8a9c9ebb68d8bc2e71 P e6001a1b382847169cac813c03722887 [term 1 FOLLOWER]: Leader election won for term 1
15:06:29.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.150964 31297 raft_consensus.cc:695] T c0712e7c0e614a8a9c9ebb68d8bc2e71 P e6001a1b382847169cac813c03722887 [term 1 LEADER]: Becoming Leader. State: Replica: e6001a1b382847169cac813c03722887, State: Running, Role: LEADER
15:06:29.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.151279 31297 consensus_queue.cc:237] T c0712e7c0e614a8a9c9ebb68d8bc2e71 P e6001a1b382847169cac813c03722887 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6001a1b382847169cac813c03722887" member_type: VOTER last_known_addr { host: "127.29.207.193" port: 39647 } } peers { permanent_uuid: "aff4ff4877ed42f58b9a3f6430894003" member_type: VOTER last_known_addr { host: "127.29.207.194" port: 36203 } } peers { permanent_uuid: "57641dcf709949098d39f0e8b0fbe6a7" member_type: VOTER last_known_addr { host: "127.29.207.195" port: 40677 } }
15:06:29.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.152094 30929 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "68b0a288f3184acfb28ff0691b224244" candidate_uuid: "57641dcf709949098d39f0e8b0fbe6a7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e6001a1b382847169cac813c03722887" is_pre_election: true
15:06:29.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.152247 30929 raft_consensus.cc:2463] T 68b0a288f3184acfb28ff0691b224244 P e6001a1b382847169cac813c03722887 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 57641dcf709949098d39f0e8b0fbe6a7 in term 0.
15:06:29.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.153287 31146 leader_election.cc:304] T 68b0a288f3184acfb28ff0691b224244 P 57641dcf709949098d39f0e8b0fbe6a7 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 57641dcf709949098d39f0e8b0fbe6a7, e6001a1b382847169cac813c03722887; no voters:
15:06:29.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.153723 31290 raft_consensus.cc:2798] T 68b0a288f3184acfb28ff0691b224244 P 57641dcf709949098d39f0e8b0fbe6a7 [term 0 FOLLOWER]: Leader pre-election won for term 1
15:06:29.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.153782 31290 raft_consensus.cc:491] T 68b0a288f3184acfb28ff0691b224244 P 57641dcf709949098d39f0e8b0fbe6a7 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
15:06:29.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.153820 31290 raft_consensus.cc:3054] T 68b0a288f3184acfb28ff0691b224244 P 57641dcf709949098d39f0e8b0fbe6a7 [term 0 FOLLOWER]: Advancing to term 1
15:06:29.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.154417 31071 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "68b0a288f3184acfb28ff0691b224244" candidate_uuid: "57641dcf709949098d39f0e8b0fbe6a7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "aff4ff4877ed42f58b9a3f6430894003" is_pre_election: true
15:06:29.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.154569 31071 raft_consensus.cc:2463] T 68b0a288f3184acfb28ff0691b224244 P aff4ff4877ed42f58b9a3f6430894003 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 57641dcf709949098d39f0e8b0fbe6a7 in term 0.
15:06:29.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.154330 30734 catalog_manager.cc:5526] T c0712e7c0e614a8a9c9ebb68d8bc2e71 P e6001a1b382847169cac813c03722887 reported cstate change: term changed from 0 to 1, leader changed from <none> to e6001a1b382847169cac813c03722887 (127.29.207.193). New cstate: current_term: 1 leader_uuid: "e6001a1b382847169cac813c03722887" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6001a1b382847169cac813c03722887" member_type: VOTER last_known_addr { host: "127.29.207.193" port: 39647 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "aff4ff4877ed42f58b9a3f6430894003" member_type: VOTER last_known_addr { host: "127.29.207.194" port: 36203 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "57641dcf709949098d39f0e8b0fbe6a7" member_type: VOTER last_known_addr { host: "127.29.207.195" port: 40677 } health_report { overall_health: UNKNOWN } } }
15:06:29.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.154829 31290 raft_consensus.cc:513] T 68b0a288f3184acfb28ff0691b224244 P 57641dcf709949098d39f0e8b0fbe6a7 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6001a1b382847169cac813c03722887" member_type: VOTER last_known_addr { host: "127.29.207.193" port: 39647 } } peers { permanent_uuid: "57641dcf709949098d39f0e8b0fbe6a7" member_type: VOTER last_known_addr { host: "127.29.207.195" port: 40677 } } peers { permanent_uuid: "aff4ff4877ed42f58b9a3f6430894003" member_type: VOTER last_known_addr { host: "127.29.207.194" port: 36203 } }
15:06:29.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.155146 31290 leader_election.cc:290] T 68b0a288f3184acfb28ff0691b224244 P 57641dcf709949098d39f0e8b0fbe6a7 [CANDIDATE]: Term 1 election: Requested vote from peers e6001a1b382847169cac813c03722887 (127.29.207.193:39647), aff4ff4877ed42f58b9a3f6430894003 (127.29.207.194:36203)
15:06:29.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.156181 30929 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "68b0a288f3184acfb28ff0691b224244" candidate_uuid: "57641dcf709949098d39f0e8b0fbe6a7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e6001a1b382847169cac813c03722887"
15:06:29.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.156309 30929 raft_consensus.cc:3054] T 68b0a288f3184acfb28ff0691b224244 P e6001a1b382847169cac813c03722887 [term 0 FOLLOWER]: Advancing to term 1
15:06:29.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.156591 31071 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "68b0a288f3184acfb28ff0691b224244" candidate_uuid: "57641dcf709949098d39f0e8b0fbe6a7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "aff4ff4877ed42f58b9a3f6430894003"
15:06:29.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.156703 31071 raft_consensus.cc:3054] T 68b0a288f3184acfb28ff0691b224244 P aff4ff4877ed42f58b9a3f6430894003 [term 0 FOLLOWER]: Advancing to term 1
15:06:29.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.157375 30929 raft_consensus.cc:2463] T 68b0a288f3184acfb28ff0691b224244 P e6001a1b382847169cac813c03722887 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 57641dcf709949098d39f0e8b0fbe6a7 in term 1.
15:06:29.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.157735 31146 leader_election.cc:304] T 68b0a288f3184acfb28ff0691b224244 P 57641dcf709949098d39f0e8b0fbe6a7 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 57641dcf709949098d39f0e8b0fbe6a7, e6001a1b382847169cac813c03722887; no voters:
15:06:29.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.157902 31290 raft_consensus.cc:2798] T 68b0a288f3184acfb28ff0691b224244 P 57641dcf709949098d39f0e8b0fbe6a7 [term 1 FOLLOWER]: Leader election won for term 1
15:06:29.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.158174 31290 raft_consensus.cc:695] T 68b0a288f3184acfb28ff0691b224244 P 57641dcf709949098d39f0e8b0fbe6a7 [term 1 LEADER]: Becoming Leader. State: Replica: 57641dcf709949098d39f0e8b0fbe6a7, State: Running, Role: LEADER
15:06:29.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.158538 31290 consensus_queue.cc:237] T 68b0a288f3184acfb28ff0691b224244 P 57641dcf709949098d39f0e8b0fbe6a7 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6001a1b382847169cac813c03722887" member_type: VOTER last_known_addr { host: "127.29.207.193" port: 39647 } } peers { permanent_uuid: "57641dcf709949098d39f0e8b0fbe6a7" member_type: VOTER last_known_addr { host: "127.29.207.195" port: 40677 } } peers { permanent_uuid: "aff4ff4877ed42f58b9a3f6430894003" member_type: VOTER last_known_addr { host: "127.29.207.194" port: 36203 } }
15:06:29.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.159993 31071 raft_consensus.cc:2463] T 68b0a288f3184acfb28ff0691b224244 P aff4ff4877ed42f58b9a3f6430894003 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 57641dcf709949098d39f0e8b0fbe6a7 in term 1.
15:06:29.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.163110 30734 catalog_manager.cc:5526] T 68b0a288f3184acfb28ff0691b224244 P 57641dcf709949098d39f0e8b0fbe6a7 reported cstate change: term changed from 0 to 1, leader changed from <none> to 57641dcf709949098d39f0e8b0fbe6a7 (127.29.207.195). New cstate: current_term: 1 leader_uuid: "57641dcf709949098d39f0e8b0fbe6a7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6001a1b382847169cac813c03722887" member_type: VOTER last_known_addr { host: "127.29.207.193" port: 39647 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "57641dcf709949098d39f0e8b0fbe6a7" member_type: VOTER last_known_addr { host: "127.29.207.195" port: 40677 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "aff4ff4877ed42f58b9a3f6430894003" member_type: VOTER last_known_addr { host: "127.29.207.194" port: 36203 } health_report { overall_health: UNKNOWN } } }
15:06:29.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.193192 30734 catalog_manager.cc:1909] Servicing CreateTable request from {username='slave'} at 127.0.0.1:44264:
15:06:29.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
15:06:29.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
15:06:29.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:06:29.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
15:06:29.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
15:06:29.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
15:06:29.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
15:06:29.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
15:06:29.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
15:06:29.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
15:06:29.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
15:06:29.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:06:29.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:06:29.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
15:06:29.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
15:06:29.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
15:06:29.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
15:06:29.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
15:06:29.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
15:06:29.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
15:06:29.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
15:06:29.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:06:29.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:06:29.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
15:06:29.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
15:06:29.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
15:06:29.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
15:06:29.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
15:06:29.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
15:06:29.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
15:06:29.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
15:06:29.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:06:29.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:06:29.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
15:06:29.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
15:06:29.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
15:06:29.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
15:06:29.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
15:06:29.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
15:06:29.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
15:06:29.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
15:06:29.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:06:29.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:06:29.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
15:06:29.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
15:06:29.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
15:06:29.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
15:06:29.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:06:29.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
15:06:29.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
15:06:29.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:06:29.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
15:06:29.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:06:29.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:06:29.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:06:29.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
15:06:29.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:29.193841 30734 catalog_manager.cc:6885] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
15:06:29.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.201784 30909 tablet_service.cc:1467] Processing CreateTablet for tablet f55bf2b487554de69951e7a3eb3dc955 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=49c3be4750994b50904bfce2fa8f396a]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
15:06:29.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.202023 31193 tablet_service.cc:1467] Processing CreateTablet for tablet f55bf2b487554de69951e7a3eb3dc955 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=49c3be4750994b50904bfce2fa8f396a]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
15:06:29.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.202481 30909 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f55bf2b487554de69951e7a3eb3dc955. 1 dirs total, 0 dirs full, 0 dirs failed
15:06:29.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.202533 31193 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f55bf2b487554de69951e7a3eb3dc955. 1 dirs total, 0 dirs full, 0 dirs failed
15:06:29.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.204738 31288 tablet_bootstrap.cc:492] T f55bf2b487554de69951e7a3eb3dc955 P 57641dcf709949098d39f0e8b0fbe6a7: Bootstrap starting.
15:06:29.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.204749 31293 tablet_bootstrap.cc:492] T f55bf2b487554de69951e7a3eb3dc955 P e6001a1b382847169cac813c03722887: Bootstrap starting.
15:06:29.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.205180 31051 tablet_service.cc:1467] Processing CreateTablet for tablet f55bf2b487554de69951e7a3eb3dc955 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=49c3be4750994b50904bfce2fa8f396a]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
15:06:29.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.205798 31293 tablet_bootstrap.cc:654] T f55bf2b487554de69951e7a3eb3dc955 P e6001a1b382847169cac813c03722887: Neither blocks nor log segments found. Creating new log.
15:06:29.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.205866 31051 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f55bf2b487554de69951e7a3eb3dc955. 1 dirs total, 0 dirs full, 0 dirs failed
15:06:29.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.206013 31288 tablet_bootstrap.cc:654] T f55bf2b487554de69951e7a3eb3dc955 P 57641dcf709949098d39f0e8b0fbe6a7: Neither blocks nor log segments found. Creating new log.
15:06:29.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.207036 31293 tablet_bootstrap.cc:492] T f55bf2b487554de69951e7a3eb3dc955 P e6001a1b382847169cac813c03722887: No bootstrap required, opened a new log
15:06:29.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.207139 31293 ts_tablet_manager.cc:1397] T f55bf2b487554de69951e7a3eb3dc955 P e6001a1b382847169cac813c03722887: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
15:06:29.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.207180 31288 tablet_bootstrap.cc:492] T f55bf2b487554de69951e7a3eb3dc955 P 57641dcf709949098d39f0e8b0fbe6a7: No bootstrap required, opened a new log
15:06:29.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.207252 31288 ts_tablet_manager.cc:1397] T f55bf2b487554de69951e7a3eb3dc955 P 57641dcf709949098d39f0e8b0fbe6a7: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
15:06:29.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.207614 31293 raft_consensus.cc:357] T f55bf2b487554de69951e7a3eb3dc955 P e6001a1b382847169cac813c03722887 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6001a1b382847169cac813c03722887" member_type: VOTER last_known_addr { host: "127.29.207.193" port: 39647 } } peers { permanent_uuid: "aff4ff4877ed42f58b9a3f6430894003" member_type: VOTER last_known_addr { host: "127.29.207.194" port: 36203 } } peers { permanent_uuid: "57641dcf709949098d39f0e8b0fbe6a7" member_type: VOTER last_known_addr { host: "127.29.207.195" port: 40677 } }
15:06:29.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.207651 31288 raft_consensus.cc:357] T f55bf2b487554de69951e7a3eb3dc955 P 57641dcf709949098d39f0e8b0fbe6a7 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6001a1b382847169cac813c03722887" member_type: VOTER last_known_addr { host: "127.29.207.193" port: 39647 } } peers { permanent_uuid: "aff4ff4877ed42f58b9a3f6430894003" member_type: VOTER last_known_addr { host: "127.29.207.194" port: 36203 } } peers { permanent_uuid: "57641dcf709949098d39f0e8b0fbe6a7" member_type: VOTER last_known_addr { host: "127.29.207.195" port: 40677 } }
15:06:29.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.207726 31293 raft_consensus.cc:383] T f55bf2b487554de69951e7a3eb3dc955 P e6001a1b382847169cac813c03722887 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:06:29.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.207734 31288 raft_consensus.cc:383] T f55bf2b487554de69951e7a3eb3dc955 P 57641dcf709949098d39f0e8b0fbe6a7 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:06:29.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.207762 31293 raft_consensus.cc:738] T f55bf2b487554de69951e7a3eb3dc955 P e6001a1b382847169cac813c03722887 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e6001a1b382847169cac813c03722887, State: Initialized, Role: FOLLOWER
15:06:29.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.207765 31288 raft_consensus.cc:738] T f55bf2b487554de69951e7a3eb3dc955 P 57641dcf709949098d39f0e8b0fbe6a7 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 57641dcf709949098d39f0e8b0fbe6a7, State: Initialized, Role: FOLLOWER
15:06:29.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.207892 31293 consensus_queue.cc:260] T f55bf2b487554de69951e7a3eb3dc955 P e6001a1b382847169cac813c03722887 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6001a1b382847169cac813c03722887" member_type: VOTER last_known_addr { host: "127.29.207.193" port: 39647 } } peers { permanent_uuid: "aff4ff4877ed42f58b9a3f6430894003" member_type: VOTER last_known_addr { host: "127.29.207.194" port: 36203 } } peers { permanent_uuid: "57641dcf709949098d39f0e8b0fbe6a7" member_type: VOTER last_known_addr { host: "127.29.207.195" port: 40677 } }
15:06:29.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.207891 31288 consensus_queue.cc:260] T f55bf2b487554de69951e7a3eb3dc955 P 57641dcf709949098d39f0e8b0fbe6a7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6001a1b382847169cac813c03722887" member_type: VOTER last_known_addr { host: "127.29.207.193" port: 39647 } } peers { permanent_uuid: "aff4ff4877ed42f58b9a3f6430894003" member_type: VOTER last_known_addr { host: "127.29.207.194" port: 36203 } } peers { permanent_uuid: "57641dcf709949098d39f0e8b0fbe6a7" member_type: VOTER last_known_addr { host: "127.29.207.195" port: 40677 } }
15:06:29.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.208140 31288 ts_tablet_manager.cc:1428] T f55bf2b487554de69951e7a3eb3dc955 P 57641dcf709949098d39f0e8b0fbe6a7: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
15:06:29.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.208150 31297 tablet_replica.cc:440] T f55bf2b487554de69951e7a3eb3dc955 P e6001a1b382847169cac813c03722887: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6001a1b382847169cac813c03722887" member_type: VOTER last_known_addr { host: "127.29.207.193" port: 39647 } } peers { permanent_uuid: "aff4ff4877ed42f58b9a3f6430894003" member_type: VOTER last_known_addr { host: "127.29.207.194" port: 36203 } } peers { permanent_uuid: "57641dcf709949098d39f0e8b0fbe6a7" member_type: VOTER last_known_addr { host: "127.29.207.195" port: 40677 } } }
15:06:29.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.208287 31297 tablet_replica.cc:443] T f55bf2b487554de69951e7a3eb3dc955 P e6001a1b382847169cac813c03722887: This TxnStatusTablet replica's current role is: FOLLOWER
15:06:29.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.208827 31294 tablet_bootstrap.cc:492] T f55bf2b487554de69951e7a3eb3dc955 P aff4ff4877ed42f58b9a3f6430894003: Bootstrap starting.
15:06:29.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.208150 31293 ts_tablet_manager.cc:1428] T f55bf2b487554de69951e7a3eb3dc955 P e6001a1b382847169cac813c03722887: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
15:06:29.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.209887 31294 tablet_bootstrap.cc:654] T f55bf2b487554de69951e7a3eb3dc955 P aff4ff4877ed42f58b9a3f6430894003: Neither blocks nor log segments found. Creating new log.
15:06:29.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.209877 31304 tablet_replica.cc:440] T f55bf2b487554de69951e7a3eb3dc955 P 57641dcf709949098d39f0e8b0fbe6a7: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6001a1b382847169cac813c03722887" member_type: VOTER last_known_addr { host: "127.29.207.193" port: 39647 } } peers { permanent_uuid: "aff4ff4877ed42f58b9a3f6430894003" member_type: VOTER last_known_addr { host: "127.29.207.194" port: 36203 } } peers { permanent_uuid: "57641dcf709949098d39f0e8b0fbe6a7" member_type: VOTER last_known_addr { host: "127.29.207.195" port: 40677 } } }
15:06:29.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.210026 31304 tablet_replica.cc:443] T f55bf2b487554de69951e7a3eb3dc955 P 57641dcf709949098d39f0e8b0fbe6a7: This TxnStatusTablet replica's current role is: FOLLOWER
15:06:29.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.211563 31294 tablet_bootstrap.cc:492] T f55bf2b487554de69951e7a3eb3dc955 P aff4ff4877ed42f58b9a3f6430894003: No bootstrap required, opened a new log
15:06:29.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.211659 31294 ts_tablet_manager.cc:1397] T f55bf2b487554de69951e7a3eb3dc955 P aff4ff4877ed42f58b9a3f6430894003: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.000s
15:06:29.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.212164 31294 raft_consensus.cc:357] T f55bf2b487554de69951e7a3eb3dc955 P aff4ff4877ed42f58b9a3f6430894003 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6001a1b382847169cac813c03722887" member_type: VOTER last_known_addr { host: "127.29.207.193" port: 39647 } } peers { permanent_uuid: "aff4ff4877ed42f58b9a3f6430894003" member_type: VOTER last_known_addr { host: "127.29.207.194" port: 36203 } } peers { permanent_uuid: "57641dcf709949098d39f0e8b0fbe6a7" member_type: VOTER last_known_addr { host: "127.29.207.195" port: 40677 } }
15:06:29.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.212276 31294 raft_consensus.cc:383] T f55bf2b487554de69951e7a3eb3dc955 P aff4ff4877ed42f58b9a3f6430894003 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:06:29.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.212311 31294 raft_consensus.cc:738] T f55bf2b487554de69951e7a3eb3dc955 P aff4ff4877ed42f58b9a3f6430894003 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: aff4ff4877ed42f58b9a3f6430894003, State: Initialized, Role: FOLLOWER
15:06:29.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.212430 31294 consensus_queue.cc:260] T f55bf2b487554de69951e7a3eb3dc955 P aff4ff4877ed42f58b9a3f6430894003 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6001a1b382847169cac813c03722887" member_type: VOTER last_known_addr { host: "127.29.207.193" port: 39647 } } peers { permanent_uuid: "aff4ff4877ed42f58b9a3f6430894003" member_type: VOTER last_known_addr { host: "127.29.207.194" port: 36203 } } peers { permanent_uuid: "57641dcf709949098d39f0e8b0fbe6a7" member_type: VOTER last_known_addr { host: "127.29.207.195" port: 40677 } }
15:06:29.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.212702 31298 tablet_replica.cc:440] T f55bf2b487554de69951e7a3eb3dc955 P aff4ff4877ed42f58b9a3f6430894003: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6001a1b382847169cac813c03722887" member_type: VOTER last_known_addr { host: "127.29.207.193" port: 39647 } } peers { permanent_uuid: "aff4ff4877ed42f58b9a3f6430894003" member_type: VOTER last_known_addr { host: "127.29.207.194" port: 36203 } } peers { permanent_uuid: "57641dcf709949098d39f0e8b0fbe6a7" member_type: VOTER last_known_addr { host: "127.29.207.195" port: 40677 } } }
15:06:29.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.212806 31298 tablet_replica.cc:443] T f55bf2b487554de69951e7a3eb3dc955 P aff4ff4877ed42f58b9a3f6430894003: This TxnStatusTablet replica's current role is: FOLLOWER
15:06:29.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.214057 31294 ts_tablet_manager.cc:1428] T f55bf2b487554de69951e7a3eb3dc955 P aff4ff4877ed42f58b9a3f6430894003: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
15:06:29.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.227141 31297 raft_consensus.cc:491] T f55bf2b487554de69951e7a3eb3dc955 P e6001a1b382847169cac813c03722887 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:06:29.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.227263 31297 raft_consensus.cc:513] T f55bf2b487554de69951e7a3eb3dc955 P e6001a1b382847169cac813c03722887 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6001a1b382847169cac813c03722887" member_type: VOTER last_known_addr { host: "127.29.207.193" port: 39647 } } peers { permanent_uuid: "aff4ff4877ed42f58b9a3f6430894003" member_type: VOTER last_known_addr { host: "127.29.207.194" port: 36203 } } peers { permanent_uuid: "57641dcf709949098d39f0e8b0fbe6a7" member_type: VOTER last_known_addr { host: "127.29.207.195" port: 40677 } }
15:06:29.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.227633 31297 leader_election.cc:290] T f55bf2b487554de69951e7a3eb3dc955 P e6001a1b382847169cac813c03722887 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers aff4ff4877ed42f58b9a3f6430894003 (127.29.207.194:36203), 57641dcf709949098d39f0e8b0fbe6a7 (127.29.207.195:40677)
15:06:29.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.228112 31071 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f55bf2b487554de69951e7a3eb3dc955" candidate_uuid: "e6001a1b382847169cac813c03722887" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "aff4ff4877ed42f58b9a3f6430894003" is_pre_election: true
15:06:29.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.228108 31214 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f55bf2b487554de69951e7a3eb3dc955" candidate_uuid: "e6001a1b382847169cac813c03722887" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "57641dcf709949098d39f0e8b0fbe6a7" is_pre_election: true
15:06:29.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.228263 31071 raft_consensus.cc:2463] T f55bf2b487554de69951e7a3eb3dc955 P aff4ff4877ed42f58b9a3f6430894003 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e6001a1b382847169cac813c03722887 in term 0.
15:06:29.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.228263 31214 raft_consensus.cc:2463] T f55bf2b487554de69951e7a3eb3dc955 P 57641dcf709949098d39f0e8b0fbe6a7 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e6001a1b382847169cac813c03722887 in term 0.
15:06:29.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.228576 30862 leader_election.cc:304] T f55bf2b487554de69951e7a3eb3dc955 P e6001a1b382847169cac813c03722887 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: aff4ff4877ed42f58b9a3f6430894003, e6001a1b382847169cac813c03722887; no voters:
15:06:29.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.228770 31297 raft_consensus.cc:2798] T f55bf2b487554de69951e7a3eb3dc955 P e6001a1b382847169cac813c03722887 [term 0 FOLLOWER]: Leader pre-election won for term 1
15:06:29.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.228827 31297 raft_consensus.cc:491] T f55bf2b487554de69951e7a3eb3dc955 P e6001a1b382847169cac813c03722887 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
15:06:29.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.228854 31297 raft_consensus.cc:3054] T f55bf2b487554de69951e7a3eb3dc955 P e6001a1b382847169cac813c03722887 [term 0 FOLLOWER]: Advancing to term 1
15:06:29.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.229584 31297 raft_consensus.cc:513] T f55bf2b487554de69951e7a3eb3dc955 P e6001a1b382847169cac813c03722887 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6001a1b382847169cac813c03722887" member_type: VOTER last_known_addr { host: "127.29.207.193" port: 39647 } } peers { permanent_uuid: "aff4ff4877ed42f58b9a3f6430894003" member_type: VOTER last_known_addr { host: "127.29.207.194" port: 36203 } } peers { permanent_uuid: "57641dcf709949098d39f0e8b0fbe6a7" member_type: VOTER last_known_addr { host: "127.29.207.195" port: 40677 } }
15:06:29.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.229871 31297 leader_election.cc:290] T f55bf2b487554de69951e7a3eb3dc955 P e6001a1b382847169cac813c03722887 [CANDIDATE]: Term 1 election: Requested vote from peers aff4ff4877ed42f58b9a3f6430894003 (127.29.207.194:36203), 57641dcf709949098d39f0e8b0fbe6a7 (127.29.207.195:40677)
15:06:29.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.230228 31214 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f55bf2b487554de69951e7a3eb3dc955" candidate_uuid: "e6001a1b382847169cac813c03722887" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "57641dcf709949098d39f0e8b0fbe6a7"
15:06:29.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.230228 31071 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f55bf2b487554de69951e7a3eb3dc955" candidate_uuid: "e6001a1b382847169cac813c03722887" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "aff4ff4877ed42f58b9a3f6430894003"
15:06:29.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.230330 31071 raft_consensus.cc:3054] T f55bf2b487554de69951e7a3eb3dc955 P aff4ff4877ed42f58b9a3f6430894003 [term 0 FOLLOWER]: Advancing to term 1
15:06:29.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.230355 31214 raft_consensus.cc:3054] T f55bf2b487554de69951e7a3eb3dc955 P 57641dcf709949098d39f0e8b0fbe6a7 [term 0 FOLLOWER]: Advancing to term 1
15:06:29.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.231129 31071 raft_consensus.cc:2463] T f55bf2b487554de69951e7a3eb3dc955 P aff4ff4877ed42f58b9a3f6430894003 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e6001a1b382847169cac813c03722887 in term 1.
15:06:29.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.231174 31214 raft_consensus.cc:2463] T f55bf2b487554de69951e7a3eb3dc955 P 57641dcf709949098d39f0e8b0fbe6a7 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e6001a1b382847169cac813c03722887 in term 1.
15:06:29.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.231441 30862 leader_election.cc:304] T f55bf2b487554de69951e7a3eb3dc955 P e6001a1b382847169cac813c03722887 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: aff4ff4877ed42f58b9a3f6430894003, e6001a1b382847169cac813c03722887; no voters:
15:06:29.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.231624 31297 raft_consensus.cc:2798] T f55bf2b487554de69951e7a3eb3dc955 P e6001a1b382847169cac813c03722887 [term 1 FOLLOWER]: Leader election won for term 1
15:06:29.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.231709 31297 raft_consensus.cc:695] T f55bf2b487554de69951e7a3eb3dc955 P e6001a1b382847169cac813c03722887 [term 1 LEADER]: Becoming Leader. State: Replica: e6001a1b382847169cac813c03722887, State: Running, Role: LEADER
15:06:29.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.231851 31297 consensus_queue.cc:237] T f55bf2b487554de69951e7a3eb3dc955 P e6001a1b382847169cac813c03722887 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6001a1b382847169cac813c03722887" member_type: VOTER last_known_addr { host: "127.29.207.193" port: 39647 } } peers { permanent_uuid: "aff4ff4877ed42f58b9a3f6430894003" member_type: VOTER last_known_addr { host: "127.29.207.194" port: 36203 } } peers { permanent_uuid: "57641dcf709949098d39f0e8b0fbe6a7" member_type: VOTER last_known_addr { host: "127.29.207.195" port: 40677 } }
15:06:29.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.232528 31302 tablet_replica.cc:440] T f55bf2b487554de69951e7a3eb3dc955 P e6001a1b382847169cac813c03722887: TxnStatusTablet state changed. Reason: New leader e6001a1b382847169cac813c03722887. Latest consensus state: current_term: 1 leader_uuid: "e6001a1b382847169cac813c03722887" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6001a1b382847169cac813c03722887" member_type: VOTER last_known_addr { host: "127.29.207.193" port: 39647 } } peers { permanent_uuid: "aff4ff4877ed42f58b9a3f6430894003" member_type: VOTER last_known_addr { host: "127.29.207.194" port: 36203 } } peers { permanent_uuid: "57641dcf709949098d39f0e8b0fbe6a7" member_type: VOTER last_known_addr { host: "127.29.207.195" port: 40677 } } }
15:06:29.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.232640 31302 tablet_replica.cc:443] T f55bf2b487554de69951e7a3eb3dc955 P e6001a1b382847169cac813c03722887: This TxnStatusTablet replica's current role is: LEADER
15:06:29.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.233374 31318 txn_status_manager.cc:874] Waiting until node catch up with all replicated operations in previous term...
15:06:29.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.233680 31318 txn_status_manager.cc:930] Loading transaction status metadata into memory...
15:06:29.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.233945 30734 catalog_manager.cc:5526] T f55bf2b487554de69951e7a3eb3dc955 P e6001a1b382847169cac813c03722887 reported cstate change: term changed from 0 to 1, leader changed from <none> to e6001a1b382847169cac813c03722887 (127.29.207.193). New cstate: current_term: 1 leader_uuid: "e6001a1b382847169cac813c03722887" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6001a1b382847169cac813c03722887" member_type: VOTER last_known_addr { host: "127.29.207.193" port: 39647 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "aff4ff4877ed42f58b9a3f6430894003" member_type: VOTER last_known_addr { host: "127.29.207.194" port: 36203 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "57641dcf709949098d39f0e8b0fbe6a7" member_type: VOTER last_known_addr { host: "127.29.207.195" port: 40677 } health_report { overall_health: UNKNOWN } } }
15:06:29.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:29.248176 31270 tablet.cc:2367] T 68b0a288f3184acfb28ff0691b224244 P 57641dcf709949098d39f0e8b0fbe6a7: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:06:29.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.251884 31071 raft_consensus.cc:1270] T f55bf2b487554de69951e7a3eb3dc955 P aff4ff4877ed42f58b9a3f6430894003 [term 1 FOLLOWER]: Refusing update from remote peer e6001a1b382847169cac813c03722887: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
15:06:29.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.252558 31214 raft_consensus.cc:1270] T f55bf2b487554de69951e7a3eb3dc955 P 57641dcf709949098d39f0e8b0fbe6a7 [term 1 FOLLOWER]: Refusing update from remote peer e6001a1b382847169cac813c03722887: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
15:06:29.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.252532 31302 consensus_queue.cc:1035] T f55bf2b487554de69951e7a3eb3dc955 P e6001a1b382847169cac813c03722887 [LEADER]: Connected to new peer: Peer: permanent_uuid: "aff4ff4877ed42f58b9a3f6430894003" member_type: VOTER last_known_addr { host: "127.29.207.194" port: 36203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:06:29.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.253059 31302 consensus_queue.cc:1035] T f55bf2b487554de69951e7a3eb3dc955 P e6001a1b382847169cac813c03722887 [LEADER]: Connected to new peer: Peer: permanent_uuid: "57641dcf709949098d39f0e8b0fbe6a7" member_type: VOTER last_known_addr { host: "127.29.207.195" port: 40677 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:06:29.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.255801 31304 tablet_replica.cc:440] T f55bf2b487554de69951e7a3eb3dc955 P 57641dcf709949098d39f0e8b0fbe6a7: TxnStatusTablet state changed. Reason: New leader e6001a1b382847169cac813c03722887. Latest consensus state: current_term: 1 leader_uuid: "e6001a1b382847169cac813c03722887" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6001a1b382847169cac813c03722887" member_type: VOTER last_known_addr { host: "127.29.207.193" port: 39647 } } peers { permanent_uuid: "aff4ff4877ed42f58b9a3f6430894003" member_type: VOTER last_known_addr { host: "127.29.207.194" port: 36203 } } peers { permanent_uuid: "57641dcf709949098d39f0e8b0fbe6a7" member_type: VOTER last_known_addr { host: "127.29.207.195" port: 40677 } } }
15:06:29.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.255925 31304 tablet_replica.cc:443] T f55bf2b487554de69951e7a3eb3dc955 P 57641dcf709949098d39f0e8b0fbe6a7: This TxnStatusTablet replica's current role is: FOLLOWER
15:06:29.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.256160 31298 tablet_replica.cc:440] T f55bf2b487554de69951e7a3eb3dc955 P aff4ff4877ed42f58b9a3f6430894003: TxnStatusTablet state changed. Reason: New leader e6001a1b382847169cac813c03722887. Latest consensus state: current_term: 1 leader_uuid: "e6001a1b382847169cac813c03722887" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6001a1b382847169cac813c03722887" member_type: VOTER last_known_addr { host: "127.29.207.193" port: 39647 } } peers { permanent_uuid: "aff4ff4877ed42f58b9a3f6430894003" member_type: VOTER last_known_addr { host: "127.29.207.194" port: 36203 } } peers { permanent_uuid: "57641dcf709949098d39f0e8b0fbe6a7" member_type: VOTER last_known_addr { host: "127.29.207.195" port: 40677 } } }
15:06:29.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.256261 31298 tablet_replica.cc:443] T f55bf2b487554de69951e7a3eb3dc955 P aff4ff4877ed42f58b9a3f6430894003: This TxnStatusTablet replica's current role is: FOLLOWER
15:06:29.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.259688 31302 tablet_replica.cc:440] T f55bf2b487554de69951e7a3eb3dc955 P e6001a1b382847169cac813c03722887: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "e6001a1b382847169cac813c03722887" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6001a1b382847169cac813c03722887" member_type: VOTER last_known_addr { host: "127.29.207.193" port: 39647 } } peers { permanent_uuid: "aff4ff4877ed42f58b9a3f6430894003" member_type: VOTER last_known_addr { host: "127.29.207.194" port: 36203 } } peers { permanent_uuid: "57641dcf709949098d39f0e8b0fbe6a7" member_type: VOTER last_known_addr { host: "127.29.207.195" port: 40677 } } }
15:06:29.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.259793 31302 tablet_replica.cc:443] T f55bf2b487554de69951e7a3eb3dc955 P e6001a1b382847169cac813c03722887: This TxnStatusTablet replica's current role is: LEADER
15:06:29.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.259896 31302 tablet_replica.cc:440] T f55bf2b487554de69951e7a3eb3dc955 P e6001a1b382847169cac813c03722887: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "e6001a1b382847169cac813c03722887" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6001a1b382847169cac813c03722887" member_type: VOTER last_known_addr { host: "127.29.207.193" port: 39647 } } peers { permanent_uuid: "aff4ff4877ed42f58b9a3f6430894003" member_type: VOTER last_known_addr { host: "127.29.207.194" port: 36203 } } peers { permanent_uuid: "57641dcf709949098d39f0e8b0fbe6a7" member_type: VOTER last_known_addr { host: "127.29.207.195" port: 40677 } } }
15:06:29.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.259946 31302 tablet_replica.cc:443] T f55bf2b487554de69951e7a3eb3dc955 P e6001a1b382847169cac813c03722887: This TxnStatusTablet replica's current role is: LEADER
15:06:29.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.260854 31304 tablet_replica.cc:440] T f55bf2b487554de69951e7a3eb3dc955 P 57641dcf709949098d39f0e8b0fbe6a7: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "e6001a1b382847169cac813c03722887" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6001a1b382847169cac813c03722887" member_type: VOTER last_known_addr { host: "127.29.207.193" port: 39647 } } peers { permanent_uuid: "aff4ff4877ed42f58b9a3f6430894003" member_type: VOTER last_known_addr { host: "127.29.207.194" port: 36203 } } peers { permanent_uuid: "57641dcf709949098d39f0e8b0fbe6a7" member_type: VOTER last_known_addr { host: "127.29.207.195" port: 40677 } } }
15:06:29.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.260941 31304 tablet_replica.cc:443] T f55bf2b487554de69951e7a3eb3dc955 P 57641dcf709949098d39f0e8b0fbe6a7: This TxnStatusTablet replica's current role is: FOLLOWER
15:06:29.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.261744 31321 mvcc.cc:204] Tried to move back new op lower bound from 7113500218369204224 to 7113500218295009280. Current Snapshot: MvccSnapshot[applied={T|T < 7113500218369204224}]
15:06:29.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.262614 31298 tablet_replica.cc:440] T f55bf2b487554de69951e7a3eb3dc955 P aff4ff4877ed42f58b9a3f6430894003: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "e6001a1b382847169cac813c03722887" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6001a1b382847169cac813c03722887" member_type: VOTER last_known_addr { host: "127.29.207.193" port: 39647 } } peers { permanent_uuid: "aff4ff4877ed42f58b9a3f6430894003" member_type: VOTER last_known_addr { host: "127.29.207.194" port: 36203 } } peers { permanent_uuid: "57641dcf709949098d39f0e8b0fbe6a7" member_type: VOTER last_known_addr { host: "127.29.207.195" port: 40677 } } }
15:06:29.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.262724 31298 tablet_replica.cc:443] T f55bf2b487554de69951e7a3eb3dc955 P aff4ff4877ed42f58b9a3f6430894003: This TxnStatusTablet replica's current role is: FOLLOWER
15:06:29.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.263669 31322 mvcc.cc:204] Tried to move back new op lower bound from 7113500218369204224 to 7113500218295009280. Current Snapshot: MvccSnapshot[applied={T|T < 7113500218369204224}]
15:06:29.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.264504 31323 mvcc.cc:204] Tried to move back new op lower bound from 7113500218369204224 to 7113500218295009280. Current Snapshot: MvccSnapshot[applied={T|T < 7113500218369204224}]
15:06:29.268 [DEBUG - Time-limited test] (KuduTransaction.java:702) starting keepalive heartbeating with period 500 ms (txn ID 0)
15:06:29.270 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table 5661f08abeeb4d1483bc1ec2b552a012
15:06:29.270 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.29.207.252:42085
15:06:29.271 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
15:06:29.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.271723 30527 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task7b0YEV/build/debug/bin/kudu with pid 30699
15:06:29.278 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.29.207.252:42085(127.29.207.252:42085)] handling channelInactive
15:06:29.278 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.207.252:42085(127.29.207.252:42085)] cleaning up while in state READY due to: connection closed
15:06:29.279 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:06:29.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.289686 30801 catalog_manager.cc:1260] Loaded cluster ID: 3fab646ada5f4cf2b1a6315d5ebfd421
15:06:29.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.289767 30801 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e: loading cluster ID for follower catalog manager: success
15:06:29.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.291458 30801 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e: acquiring CA information for follower catalog manager: success
15:06:29.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.292629 30801 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:06:29.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:29.316625 31127 tablet.cc:2367] T 68b0a288f3184acfb28ff0691b224244 P aff4ff4877ed42f58b9a3f6430894003: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:06:29.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.370175 30836 catalog_manager.cc:1260] Loaded cluster ID: 3fab646ada5f4cf2b1a6315d5ebfd421
15:06:29.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.370280 30836 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115: loading cluster ID for follower catalog manager: success
15:06:29.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.372849 30836 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115: acquiring CA information for follower catalog manager: success
15:06:29.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.374429 30836 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:06:29.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.539126 31330 raft_consensus.cc:491] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 5168bd8e804e45abae06fee65e7c306d)
15:06:29.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.539309 31330 raft_consensus.cc:513] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a86734036df442d9067f984db9f965e" member_type: VOTER last_known_addr { host: "127.29.207.254" port: 42611 } } peers { permanent_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115" member_type: VOTER last_known_addr { host: "127.29.207.253" port: 44061 } } peers { permanent_uuid: "5168bd8e804e45abae06fee65e7c306d" member_type: VOTER last_known_addr { host: "127.29.207.252" port: 42085 } }
15:06:29.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.539343 31331 raft_consensus.cc:491] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 5168bd8e804e45abae06fee65e7c306d)
15:06:29.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.539475 31331 raft_consensus.cc:513] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a86734036df442d9067f984db9f965e" member_type: VOTER last_known_addr { host: "127.29.207.254" port: 42611 } } peers { permanent_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115" member_type: VOTER last_known_addr { host: "127.29.207.253" port: 44061 } } peers { permanent_uuid: "5168bd8e804e45abae06fee65e7c306d" member_type: VOTER last_known_addr { host: "127.29.207.252" port: 42085 } }
15:06:29.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.540302 31330 leader_election.cc:290] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 80c9fc0ab13d44ffa2a33e3b8c32f115 (127.29.207.253:44061), 5168bd8e804e45abae06fee65e7c306d (127.29.207.252:42085)
15:06:29.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.540699 30659 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1a86734036df442d9067f984db9f965e" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115" is_pre_election: true
15:06:29.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.540735 30579 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "1a86734036df442d9067f984db9f965e" is_pre_election: true
15:06:29.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.540840 30659 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1a86734036df442d9067f984db9f965e in term 1.
15:06:29.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.540874 30579 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 80c9fc0ab13d44ffa2a33e3b8c32f115 in term 1.
15:06:29.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.541226 30556 leader_election.cc:304] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1a86734036df442d9067f984db9f965e, 80c9fc0ab13d44ffa2a33e3b8c32f115; no voters:
15:06:29.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.541272 31331 leader_election.cc:290] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 1a86734036df442d9067f984db9f965e (127.29.207.254:42611), 5168bd8e804e45abae06fee65e7c306d (127.29.207.252:42085)
15:06:29.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.541257 30637 leader_election.cc:304] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1a86734036df442d9067f984db9f965e, 80c9fc0ab13d44ffa2a33e3b8c32f115; no voters:
15:06:29.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.541443 31330 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e [term 1 FOLLOWER]: Leader pre-election won for term 2
15:06:29.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.541477 31331 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115 [term 1 FOLLOWER]: Leader pre-election won for term 2
15:06:29.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.541497 31330 raft_consensus.cc:491] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e [term 1 FOLLOWER]: Starting leader election (detected failure of leader 5168bd8e804e45abae06fee65e7c306d)
15:06:29.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.541523 31331 raft_consensus.cc:491] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 5168bd8e804e45abae06fee65e7c306d)
15:06:29.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.541539 31330 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e [term 1 FOLLOWER]: Advancing to term 2
15:06:29.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.541563 31331 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115 [term 1 FOLLOWER]: Advancing to term 2
15:06:29.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:29.541949 30555 leader_election.cc:336] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 5168bd8e804e45abae06fee65e7c306d (127.29.207.252:42085): Network error: Client connection negotiation failed: client connection to 127.29.207.252:42085: connect: Connection refused (error 111)
15:06:29.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.542665 31331 raft_consensus.cc:513] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a86734036df442d9067f984db9f965e" member_type: VOTER last_known_addr { host: "127.29.207.254" port: 42611 } } peers { permanent_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115" member_type: VOTER last_known_addr { host: "127.29.207.253" port: 44061 } } peers { permanent_uuid: "5168bd8e804e45abae06fee65e7c306d" member_type: VOTER last_known_addr { host: "127.29.207.252" port: 42085 } }
15:06:29.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.542680 31330 raft_consensus.cc:513] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a86734036df442d9067f984db9f965e" member_type: VOTER last_known_addr { host: "127.29.207.254" port: 42611 } } peers { permanent_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115" member_type: VOTER last_known_addr { host: "127.29.207.253" port: 44061 } } peers { permanent_uuid: "5168bd8e804e45abae06fee65e7c306d" member_type: VOTER last_known_addr { host: "127.29.207.252" port: 42085 } }
15:06:29.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:29.542935 30637 leader_election.cc:336] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 5168bd8e804e45abae06fee65e7c306d (127.29.207.252:42085): Network error: Client connection negotiation failed: client connection to 127.29.207.252:42085: connect: Connection refused (error 111)
15:06:29.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.542974 31331 leader_election.cc:290] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115 [CANDIDATE]: Term 2 election: Requested vote from peers 1a86734036df442d9067f984db9f965e (127.29.207.254:42611), 5168bd8e804e45abae06fee65e7c306d (127.29.207.252:42085)
15:06:29.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.543282 31330 leader_election.cc:290] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e [CANDIDATE]: Term 2 election: Requested vote from peers 80c9fc0ab13d44ffa2a33e3b8c32f115 (127.29.207.253:44061), 5168bd8e804e45abae06fee65e7c306d (127.29.207.252:42085)
15:06:29.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.543339 30659 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1a86734036df442d9067f984db9f965e" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115"
15:06:29.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.543484 30659 raft_consensus.cc:2388] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 1a86734036df442d9067f984db9f965e in current term 2: Already voted for candidate 80c9fc0ab13d44ffa2a33e3b8c32f115 in this term.
15:06:29.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.543653 30579 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "1a86734036df442d9067f984db9f965e"
15:06:29.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.543784 30579 raft_consensus.cc:2388] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 80c9fc0ab13d44ffa2a33e3b8c32f115 in current term 2: Already voted for candidate 1a86734036df442d9067f984db9f965e in this term.
15:06:29.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:29.544004 30555 leader_election.cc:336] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 5168bd8e804e45abae06fee65e7c306d (127.29.207.252:42085): Network error: Client connection negotiation failed: client connection to 127.29.207.252:42085: connect: Connection refused (error 111)
15:06:29.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.544059 30555 leader_election.cc:304] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1a86734036df442d9067f984db9f965e; no voters: 5168bd8e804e45abae06fee65e7c306d, 80c9fc0ab13d44ffa2a33e3b8c32f115
15:06:29.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.544265 31330 raft_consensus.cc:2743] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
15:06:29.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:29.544749 30637 leader_election.cc:336] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 5168bd8e804e45abae06fee65e7c306d (127.29.207.252:42085): Network error: Client connection negotiation failed: client connection to 127.29.207.252:42085: connect: Connection refused (error 111)
15:06:29.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.544821 30637 leader_election.cc:304] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 80c9fc0ab13d44ffa2a33e3b8c32f115; no voters: 1a86734036df442d9067f984db9f965e, 5168bd8e804e45abae06fee65e7c306d
15:06:29.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.544965 31331 raft_consensus.cc:2743] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
15:06:29.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.621459 31304 consensus_queue.cc:1035] T 68b0a288f3184acfb28ff0691b224244 P 57641dcf709949098d39f0e8b0fbe6a7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "aff4ff4877ed42f58b9a3f6430894003" member_type: VOTER last_known_addr { host: "127.29.207.194" port: 36203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:06:29.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:29.623625 31124 heartbeater.cc:643] Failed to heartbeat to 127.29.207.252:42085 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.29.207.252:42085: connect: Connection refused (error 111)
15:06:29.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.626580 31304 consensus_queue.cc:1035] T 68b0a288f3184acfb28ff0691b224244 P 57641dcf709949098d39f0e8b0fbe6a7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e6001a1b382847169cac813c03722887" member_type: VOTER last_known_addr { host: "127.29.207.193" port: 39647 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:06:29.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:29.626713 31267 heartbeater.cc:643] Failed to heartbeat to 127.29.207.252:42085 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.29.207.252:42085: connect: Connection refused (error 111)
15:06:29.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:29.629463 30982 heartbeater.cc:643] Failed to heartbeat to 127.29.207.252:42085 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.29.207.252:42085: connect: Connection refused (error 111)
15:06:29.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.629937 31326 consensus_queue.cc:1035] T c0712e7c0e614a8a9c9ebb68d8bc2e71 P e6001a1b382847169cac813c03722887 [LEADER]: Connected to new peer: Peer: permanent_uuid: "57641dcf709949098d39f0e8b0fbe6a7" member_type: VOTER last_known_addr { host: "127.29.207.195" port: 40677 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:06:29.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.634433 31297 consensus_queue.cc:1035] T c0712e7c0e614a8a9c9ebb68d8bc2e71 P e6001a1b382847169cac813c03722887 [LEADER]: Connected to new peer: Peer: permanent_uuid: "aff4ff4877ed42f58b9a3f6430894003" member_type: VOTER last_known_addr { host: "127.29.207.194" port: 36203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:06:29.780 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.29.207.252:42085(127.29.207.252:42085)] connecting to peer
15:06:29.782 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085
15:06:29.782 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.207.252:42085(127.29.207.252:42085)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085
15:06:29.782 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location master-127.29.207.252:42085(127.29.207.252:42085) for tablet Kudu Master: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085
15:06:29.800 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.29.207.252:42085(127.29.207.252:42085)] connecting to peer
15:06:29.802 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085
15:06:29.802 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.207.252:42085(127.29.207.252:42085)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085
15:06:29.802 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.207.252:42085: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085
15:06:29.802 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.29.207.254:42611,127.29.207.252:42085,127.29.207.253:44061; will retry
15:06:29.820 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.29.207.252:42085(127.29.207.252:42085)] connecting to peer
15:06:29.821 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085
15:06:29.821 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.207.252:42085(127.29.207.252:42085)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085
15:06:29.822 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.207.252:42085: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085
15:06:29.823 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.29.207.254:42611,127.29.207.252:42085,127.29.207.253:44061; will retry
15:06:29.840 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.29.207.252:42085(127.29.207.252:42085)] connecting to peer
15:06:29.842 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085
15:06:29.842 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.207.252:42085(127.29.207.252:42085)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085
15:06:29.842 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.207.252:42085: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085
15:06:29.842 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.29.207.254:42611,127.29.207.252:42085,127.29.207.253:44061; will retry
15:06:29.861 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.29.207.252:42085(127.29.207.252:42085)] connecting to peer
15:06:29.861 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085
15:06:29.862 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.207.252:42085(127.29.207.252:42085)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085
15:06:29.862 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.207.252:42085: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085
15:06:29.862 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.29.207.254:42611,127.29.207.252:42085,127.29.207.253:44061; will retry
15:06:29.881 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.29.207.252:42085(127.29.207.252:42085)] connecting to peer
15:06:29.883 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085
15:06:29.883 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.207.252:42085(127.29.207.252:42085)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085
15:06:29.883 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.207.252:42085: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085
15:06:29.883 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.29.207.254:42611,127.29.207.252:42085,127.29.207.253:44061; will retry
15:06:29.900 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.29.207.252:42085(127.29.207.252:42085)] connecting to peer
15:06:29.901 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085
15:06:29.901 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.207.252:42085(127.29.207.252:42085)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085
15:06:29.901 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.207.252:42085: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085
15:06:29.902 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.29.207.254:42611,127.29.207.252:42085,127.29.207.253:44061; will retry
15:06:29.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.917521 31330 raft_consensus.cc:491] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:06:29.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.917671 31330 raft_consensus.cc:513] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a86734036df442d9067f984db9f965e" member_type: VOTER last_known_addr { host: "127.29.207.254" port: 42611 } } peers { permanent_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115" member_type: VOTER last_known_addr { host: "127.29.207.253" port: 44061 } } peers { permanent_uuid: "5168bd8e804e45abae06fee65e7c306d" member_type: VOTER last_known_addr { host: "127.29.207.252" port: 42085 } }
15:06:29.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.918200 31330 leader_election.cc:290] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 80c9fc0ab13d44ffa2a33e3b8c32f115 (127.29.207.253:44061), 5168bd8e804e45abae06fee65e7c306d (127.29.207.252:42085)
15:06:29.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.918694 30659 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1a86734036df442d9067f984db9f965e" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115" is_pre_election: true
15:06:29.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.918856 30659 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1a86734036df442d9067f984db9f965e in term 2.
15:06:29.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.919198 30556 leader_election.cc:304] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1a86734036df442d9067f984db9f965e, 80c9fc0ab13d44ffa2a33e3b8c32f115; no voters:
15:06:29.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.919387 31330 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e [term 2 FOLLOWER]: Leader pre-election won for term 3
15:06:29.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.919435 31330 raft_consensus.cc:491] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
15:06:29.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.919472 31330 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e [term 2 FOLLOWER]: Advancing to term 3
15:06:29.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:29.919577 30555 leader_election.cc:336] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 5168bd8e804e45abae06fee65e7c306d (127.29.207.252:42085): Network error: Client connection negotiation failed: client connection to 127.29.207.252:42085: connect: Connection refused (error 111)
15:06:29.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.920351 31330 raft_consensus.cc:513] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a86734036df442d9067f984db9f965e" member_type: VOTER last_known_addr { host: "127.29.207.254" port: 42611 } } peers { permanent_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115" member_type: VOTER last_known_addr { host: "127.29.207.253" port: 44061 } } peers { permanent_uuid: "5168bd8e804e45abae06fee65e7c306d" member_type: VOTER last_known_addr { host: "127.29.207.252" port: 42085 } }
15:06:29.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.920650 31330 leader_election.cc:290] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e [CANDIDATE]: Term 3 election: Requested vote from peers 80c9fc0ab13d44ffa2a33e3b8c32f115 (127.29.207.253:44061), 5168bd8e804e45abae06fee65e7c306d (127.29.207.252:42085)
15:06:29.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.920976 30659 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1a86734036df442d9067f984db9f965e" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115"
15:06:29.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.921101 30659 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115 [term 2 FOLLOWER]: Advancing to term 3
15:06:29.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:29.921512 30555 leader_election.cc:336] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 5168bd8e804e45abae06fee65e7c306d (127.29.207.252:42085): Network error: Client connection negotiation failed: client connection to 127.29.207.252:42085: connect: Connection refused (error 111)
15:06:29.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.922017 30659 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1a86734036df442d9067f984db9f965e in term 3.
15:06:29.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.922302 30556 leader_election.cc:304] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 1a86734036df442d9067f984db9f965e, 80c9fc0ab13d44ffa2a33e3b8c32f115; no voters: 5168bd8e804e45abae06fee65e7c306d
15:06:29.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.922470 31330 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e [term 3 FOLLOWER]: Leader election won for term 3
15:06:29.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.922668 31330 raft_consensus.cc:695] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e [term 3 LEADER]: Becoming Leader. State: Replica: 1a86734036df442d9067f984db9f965e, State: Running, Role: LEADER
15:06:29.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.923004 31330 consensus_queue.cc:237] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a86734036df442d9067f984db9f965e" member_type: VOTER last_known_addr { host: "127.29.207.254" port: 42611 } } peers { permanent_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115" member_type: VOTER last_known_addr { host: "127.29.207.253" port: 44061 } } peers { permanent_uuid: "5168bd8e804e45abae06fee65e7c306d" member_type: VOTER last_known_addr { host: "127.29.207.252" port: 42085 } }
15:06:29.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.923971 31342 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1a86734036df442d9067f984db9f965e. Latest consensus state: current_term: 3 leader_uuid: "1a86734036df442d9067f984db9f965e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a86734036df442d9067f984db9f965e" member_type: VOTER last_known_addr { host: "127.29.207.254" port: 42611 } } peers { permanent_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115" member_type: VOTER last_known_addr { host: "127.29.207.253" port: 44061 } } peers { permanent_uuid: "5168bd8e804e45abae06fee65e7c306d" member_type: VOTER last_known_addr { host: "127.29.207.252" port: 42085 } } }
15:06:29.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.924074 31342 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e [sys.catalog]: This master's current role is: LEADER
15:06:29.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.924351 31343 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
15:06:29.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.925853 31343 catalog_manager.cc:670] Loaded metadata for table kudu_system.kudu_transactions [id=49c3be4750994b50904bfce2fa8f396a]
15:06:29.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.926236 31343 catalog_manager.cc:670] Loaded metadata for table txn_manager_fallback_rolling [id=5661f08abeeb4d1483bc1ec2b552a012]
15:06:29.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.927412 31343 tablet_loader.cc:96] loaded metadata for tablet 68b0a288f3184acfb28ff0691b224244 (table txn_manager_fallback_rolling [id=5661f08abeeb4d1483bc1ec2b552a012])
15:06:29.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.927659 31343 tablet_loader.cc:96] loaded metadata for tablet c0712e7c0e614a8a9c9ebb68d8bc2e71 (table txn_manager_fallback_rolling [id=5661f08abeeb4d1483bc1ec2b552a012])
15:06:29.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.927834 31343 tablet_loader.cc:96] loaded metadata for tablet f55bf2b487554de69951e7a3eb3dc955 (table kudu_system.kudu_transactions [id=49c3be4750994b50904bfce2fa8f396a])
15:06:29.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.927933 31343 catalog_manager.cc:1485] Initializing Kudu cluster ID...
15:06:29.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.928351 31343 catalog_manager.cc:1260] Loaded cluster ID: 3fab646ada5f4cf2b1a6315d5ebfd421
15:06:29.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.928397 31343 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
15:06:29.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.928890 31343 catalog_manager.cc:1505] Loading token signing keys...
15:06:29.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.929304 31343 catalog_manager.cc:5910] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e: Loaded TSK: 0
15:06:29.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:29.929996 31343 catalog_manager.cc:1515] Initializing in-progress tserver states...
15:06:30.000 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.29.207.252:42085(127.29.207.252:42085)] connecting to peer
15:06:30.001 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085
15:06:30.001 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.207.252:42085(127.29.207.252:42085)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085
15:06:30.001 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.207.252:42085: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085
15:06:30.002 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
15:06:30.003 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] sending RPC to server master-127.29.207.252:42085, [2ms] received response from server master-127.29.207.252:42085: Network error: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085, [2ms] delaying RPC due to: Network error: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.254:42611, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.252:42085, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.253:44061, [22ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.252:42085: Network error: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085, [22ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.254:42611: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.253:44061: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.29.207.254:42611,127.29.207.252:42085,127.29.207.253:44061) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.254:42611, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.252:42085, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.253:44061, [41ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.254:42611: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.252:42085: Network error: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085, [43ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.253:44061: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.29.207.254:42611,127.29.207.252:42085,127.29.207.253:44061) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.254:42611, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.252:42085, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.253:44061, [61ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.254:42611: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.252:42085: Network error: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085, [62ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.253:44061: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.29.207.254:42611,127.29.207.252:42085,127.29.207.253:44061) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.254:42611, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.252:42085, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.253:44061, [82ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.254:42611: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.252:42085: Network error: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085, [82ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.253:44061: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.29.207.254:42611,127.29.207.252:42085,127.29.207.253:44061) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.254:42611, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.252:42085, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.253:44061, [102ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.254:42611: OK, [103ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.252:42085: Network error: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085, [103ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.253:44061: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.29.207.254:42611,127.29.207.252:42085,127.29.207.253:44061) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.254:42611, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.252:42085, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.253:44061, [121ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.254:42611: OK, [121ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.252:42085: Network error: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085, [122ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.253:44061: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.29.207.254:42611,127.29.207.252:42085,127.29.207.253:44061) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085, [219ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.254:42611, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.252:42085, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.253:44061, [221ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.252:42085: Network error: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085, [222ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.253:44061: OK, [222ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.254:42611: OK, deferred=Deferred@412511858(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1027305234@2ccc581a -> (continuation of Deferred@134885086 after retry RPC after error@1896701554) -> (continuation of Deferred@1514363871 after retry RPC after error@344052741) -> (continuation of Deferred@1294230677 after retry RPC after error@1447124365) -> (continuation of Deferred@379353401 after retry RPC after error@203929107) -> (continuation of Deferred@1648982990 after retry RPC after error@1043918655) -> (continuation of Deferred@1259871344 after retry RPC after error@2059320366), errback=org.apache.kudu.client.KuduTransaction$$Lambda$84/1543002520@53003fbc -> (continuation of Deferred@134885086 after retry RPC after error@1896701554) -> (continuation of Deferred@1514363871 after retry RPC after error@344052741) -> (continuation of Deferred@1294230677 after retry RPC after error@1447124365) -> (continuation of Deferred@379353401 after retry RPC after error@203929107) -> (continuation of Deferred@1648982990 after retry RPC after error@1043918655) -> (continuation of Deferred@1259871344 after retry RPC after error@2059320366))) after lookup
15:06:30.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:30.010890 30659 raft_consensus.cc:1270] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115 [term 3 FOLLOWER]: Refusing update from remote peer 1a86734036df442d9067f984db9f965e: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 3 index: 12. (index mismatch)
15:06:30.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:30.011452 31342 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e [LEADER]: Connected to new peer: Peer: permanent_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115" member_type: VOTER last_known_addr { host: "127.29.207.253" port: 44061 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
15:06:30.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:30.012517 31331 sys_catalog.cc:455] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1a86734036df442d9067f984db9f965e. Latest consensus state: current_term: 3 leader_uuid: "1a86734036df442d9067f984db9f965e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a86734036df442d9067f984db9f965e" member_type: VOTER last_known_addr { host: "127.29.207.254" port: 42611 } } peers { permanent_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115" member_type: VOTER last_known_addr { host: "127.29.207.253" port: 44061 } } peers { permanent_uuid: "5168bd8e804e45abae06fee65e7c306d" member_type: VOTER last_known_addr { host: "127.29.207.252" port: 42085 } } }
15:06:30.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:30.012627 31331 sys_catalog.cc:458] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115 [sys.catalog]: This master's current role is: FOLLOWER
15:06:30.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:30.013648 31330 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "1a86734036df442d9067f984db9f965e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a86734036df442d9067f984db9f965e" member_type: VOTER last_known_addr { host: "127.29.207.254" port: 42611 } } peers { permanent_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115" member_type: VOTER last_known_addr { host: "127.29.207.253" port: 44061 } } peers { permanent_uuid: "5168bd8e804e45abae06fee65e7c306d" member_type: VOTER last_known_addr { host: "127.29.207.252" port: 42085 } } }
15:06:30.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:30.013756 31330 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e [sys.catalog]: This master's current role is: LEADER
15:06:30.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:30.013960 31331 sys_catalog.cc:455] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "1a86734036df442d9067f984db9f965e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a86734036df442d9067f984db9f965e" member_type: VOTER last_known_addr { host: "127.29.207.254" port: 42611 } } peers { permanent_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115" member_type: VOTER last_known_addr { host: "127.29.207.253" port: 44061 } } peers { permanent_uuid: "5168bd8e804e45abae06fee65e7c306d" member_type: VOTER last_known_addr { host: "127.29.207.252" port: 42085 } } }
15:06:30.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:30.014087 31331 sys_catalog.cc:458] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115 [sys.catalog]: This master's current role is: FOLLOWER
15:06:30.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:30.023754 30555 consensus_peers.cc:487] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e -> Peer 5168bd8e804e45abae06fee65e7c306d (127.29.207.252:42085): Couldn't send request to peer 5168bd8e804e45abae06fee65e7c306d. Status: Network error: Client connection negotiation failed: client connection to 127.29.207.252:42085: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
15:06:30.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:30.031267 30599 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:54038 (request call id 8) took 27 ms (client timeout 27 ms). Trace:
15:06:30.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:30.031365 30599 rpcz_store.cc:269] 0112 15:06:30.004076 (+ 0us) service_pool.cc:168] Inserting onto call queue
15:06:30.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:06:30.004139 (+ 63us) service_pool.cc:225] Handling call
15:06:30.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:06:30.031237 (+ 27098us) inbound_call.cc:173] Queueing success response
15:06:30.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"dns_us":406,"init.queue_time_us":215,"thread_start_us":438,"threads_started":5}
15:06:30.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:30.031738 30568 catalog_manager.cc:1909] Servicing CreateTable request from {username='slave'} at 127.0.0.1:54050:
15:06:30.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
15:06:30.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
15:06:30.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:06:30.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
15:06:30.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
15:06:30.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
15:06:30.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
15:06:30.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
15:06:30.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
15:06:30.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
15:06:30.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
15:06:30.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:06:30.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:06:30.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
15:06:30.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
15:06:30.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
15:06:30.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
15:06:30.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
15:06:30.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
15:06:30.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
15:06:30.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
15:06:30.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:06:30.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:06:30.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
15:06:30.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
15:06:30.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
15:06:30.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
15:06:30.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
15:06:30.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
15:06:30.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
15:06:30.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
15:06:30.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:06:30.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:06:30.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
15:06:30.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
15:06:30.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
15:06:30.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
15:06:30.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
15:06:30.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
15:06:30.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
15:06:30.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
15:06:30.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:06:30.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:06:30.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
15:06:30.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
15:06:30.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
15:06:30.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
15:06:30.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:06:30.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
15:06:30.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
15:06:30.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:06:30.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
15:06:30.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:06:30.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:06:30.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:06:30.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
15:06:30.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:30.033232 30568 catalog_manager.cc:6885] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
15:06:30.033 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=9, TimeoutTracker(timeout=250, elapsed=252), Traces: [0ms] sending RPC to server master-127.29.207.252:42085, [2ms] received response from server master-127.29.207.252:42085: Network error: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085, [2ms] delaying RPC due to: Network error: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.254:42611, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.252:42085, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.253:44061, [22ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.252:42085: Network error: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085, [22ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.254:42611: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.253:44061: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.29.207.254:42611,127.29.207.252:42085,127.29.207.253:44061) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.254:42611, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.252:42085, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.253:44061, [41ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.254:42611: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.252:42085: Network error: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085, [43ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.253:44061: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.29.207.254:42611,127.29.207.252:42085,127.29.207.253:44061) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.254:42611, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.252:42085, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.253:44061, [61ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.254:42611: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.252:42085: Network error: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085, [62ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.253:44061: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.29.207.254:42611,127.29.207.252:42085,127.29.207.253:44061) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.254:42611, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.252:42085, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.253:44061, [82ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.254:42611: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.252:42085: Network error: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085, [82ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.253:44061: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.29.207.254:42611,127.29.207.252:42085,127.29.207.253:44061) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.254:42611, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.252:42085, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.253:44061, [102ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.254:42611: OK, [103ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.252:42085: Network error: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085, [103ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.253:44061: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.29.207.254:42611,127.29.207.252:42085,127.29.207.253:44061) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.254:42611, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.252:42085, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.253:44061, [121ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.254:42611: OK, [121ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.252:42085: Network error: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085, [122ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.253:44061: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.29.207.254:42611,127.29.207.252:42085,127.29.207.253:44061) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085, [219ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.254:42611, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.252:42085, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.253:44061, [221ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.252:42085: Network error: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085, [222ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.253:44061: OK, [222ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.254:42611: OK, [223ms] sending RPC to server master-127.29.207.254:42611, [252ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@412511858(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1027305234@2ccc581a -> (continuation of Deferred@134885086 after retry RPC after error@1896701554) -> (continuation of Deferred@1514363871 after retry RPC after error@344052741) -> (continuation of Deferred@1294230677 after retry RPC after error@1447124365) -> (continuation of Deferred@379353401 after retry RPC after error@203929107) -> (continuation of Deferred@1648982990 after retry RPC after error@1043918655) -> (continuation of Deferred@1259871344 after retry RPC after error@2059320366) -> (continuation of Deferred@360303237 after retry RPC@1952117644), errback=org.apache.kudu.client.KuduTransaction$$Lambda$84/1543002520@53003fbc -> (continuation of Deferred@134885086 after retry RPC after error@1896701554) -> (continuation of Deferred@1514363871 after retry RPC after error@344052741) -> (continuation of Deferred@1294230677 after retry RPC after error@1447124365) -> (continuation of Deferred@379353401 after retry RPC after error@203929107) -> (continuation of Deferred@1648982990 after retry RPC after error@1043918655) -> (continuation of Deferred@1259871344 after retry RPC after error@2059320366) -> (continuation of Deferred@360303237 after retry RPC@1952117644)))
15:06:30.034 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=9, TimeoutTracker(timeout=250, elapsed=252), Traces: [0ms] sending RPC to server master-127.29.207.252:42085, [2ms] received response from server master-127.29.207.252:42085: Network error: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085, [2ms] delaying RPC due to: Network error: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.254:42611, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.252:42085, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.253:44061, [22ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.252:42085: Network error: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085, [22ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.254:42611: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.253:44061: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.29.207.254:42611,127.29.207.252:42085,127.29.207.253:44061) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.254:42611, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.252:42085, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.253:44061, [41ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.254:42611: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.252:42085: Network error: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085, [43ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.253:44061: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.29.207.254:42611,127.29.207.252:42085,127.29.207.253:44061) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.254:42611, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.252:42085, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.253:44061, [61ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.254:42611: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.252:42085: Network error: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085, [62ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.253:44061: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.29.207.254:42611,127.29.207.252:42085,127.29.207.253:44061) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.254:42611, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.252:42085, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.253:44061, [82ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.254:42611: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.252:42085: Network error: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085, [82ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.253:44061: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.29.207.254:42611,127.29.207.252:42085,127.29.207.253:44061) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.254:42611, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.252:42085, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.253:44061, [102ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.254:42611: OK, [103ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.252:42085: Network error: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085, [103ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.253:44061: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.29.207.254:42611,127.29.207.252:42085,127.29.207.253:44061) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.254:42611, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.252:42085, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.253:44061, [121ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.254:42611: OK, [121ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.252:42085: Network error: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085, [122ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.253:44061: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.29.207.254:42611,127.29.207.252:42085,127.29.207.253:44061) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085, [219ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.254:42611, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.252:42085, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.253:44061, [221ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.252:42085: Network error: Failed to connect to peer master-127.29.207.252:42085(127.29.207.252:42085): Connection refused: /127.29.207.252:42085, [222ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.253:44061: OK, [222ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.254:42611: OK, [223ms] sending RPC to server master-127.29.207.254:42611, [252ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@412511858(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1027305234@2ccc581a -> (continuation of Deferred@134885086 after retry RPC after error@1896701554) -> (continuation of Deferred@1514363871 after retry RPC after error@344052741) -> (continuation of Deferred@1294230677 after retry RPC after error@1447124365) -> (continuation of Deferred@379353401 after retry RPC after error@203929107) -> (continuation of Deferred@1648982990 after retry RPC after error@1043918655) -> (continuation of Deferred@1259871344 after retry RPC after error@2059320366) -> (continuation of Deferred@360303237 after retry RPC@1952117644), errback=org.apache.kudu.client.KuduTransaction$$Lambda$84/1543002520@53003fbc -> (continuation of Deferred@134885086 after retry RPC after error@1896701554) -> (continuation of Deferred@1514363871 after retry RPC after error@344052741) -> (continuation of Deferred@1294230677 after retry RPC after error@1447124365) -> (continuation of Deferred@379353401 after retry RPC after error@203929107) -> (continuation of Deferred@1648982990 after retry RPC after error@1043918655) -> (continuation of Deferred@1259871344 after retry RPC after error@2059320366) -> (continuation of Deferred@360303237 after retry RPC@1952117644)))
15:06:30.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:30.531167 30555 consensus_peers.cc:487] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e -> Peer 5168bd8e804e45abae06fee65e7c306d (127.29.207.252:42085): Couldn't send request to peer 5168bd8e804e45abae06fee65e7c306d. Status: Network error: Client connection negotiation failed: client connection to 127.29.207.252:42085: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
15:06:31.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.057507 31268 heartbeater.cc:502] Master 127.29.207.254:42611 was elected leader, sending a full tablet report...
15:06:31.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:31.071815 30555 consensus_peers.cc:487] T 00000000000000000000000000000000 P 1a86734036df442d9067f984db9f965e -> Peer 5168bd8e804e45abae06fee65e7c306d (127.29.207.252:42085): Couldn't send request to peer 5168bd8e804e45abae06fee65e7c306d. Status: Network error: Client connection negotiation failed: client connection to 127.29.207.252:42085: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
15:06:31.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.087209 30983 heartbeater.cc:502] Master 127.29.207.254:42611 was elected leader, sending a full tablet report...
15:06:31.284 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 68b0a288f3184acfb28ff0691b224244 for table 'txn_manager_fallback_rolling' with partition [0x00000000, 0x00000001)
15:06:31.284 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet c0712e7c0e614a8a9c9ebb68d8bc2e71 for table 'txn_manager_fallback_rolling' with partition [0x00000001, 0x00000002)
15:06:31.284 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x00000000, upperBoundPartitionKey=0x00000001, ttl=299999, tablet=68b0a288f3184acfb28ff0691b224244@[57641dcf709949098d39f0e8b0fbe6a7(127.29.207.195:40677)[L],aff4ff4877ed42f58b9a3f6430894003(127.29.207.194:36203),e6001a1b382847169cac813c03722887(127.29.207.193:39647)]}, Tablet{lowerBoundPartitionKey=0x00000001, upperBoundPartitionKey=0x00000002, ttl=299999, tablet=c0712e7c0e614a8a9c9ebb68d8bc2e71@[57641dcf709949098d39f0e8b0fbe6a7(127.29.207.195:40677),aff4ff4877ed42f58b9a3f6430894003(127.29.207.194:36203),e6001a1b382847169cac813c03722887(127.29.207.193:39647)[L]]}, NonCoveredRange{lowerBoundPartitionKey=0x00000002, upperBoundPartitionKey=0x, ttl=299999}]
15:06:31.284 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=2), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.207.254:42611, [1ms] Sub RPC GetTableLocations: received response from server master-127.29.207.254:42611: OK, deferred=null) after lookup
15:06:31.285 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 57641dcf709949098d39f0e8b0fbe6a7(127.29.207.195:40677)] connecting to peer
15:06:31.285 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 57641dcf709949098d39f0e8b0fbe6a7(127.29.207.195:40677)] Successfully connected to peer
15:06:31.296 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xa6102f1e, L:/127.0.0.1:53598 - R:/127.29.207.195:40677] using TOKEN/null
15:06:31.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:31.299283 31246 master_proxy_rpc.cc:203] Re-attempting GetTableSchema request to leader Master (127.29.207.252:42085)
15:06:31.321 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.29.207.252:42085
15:06:31.321 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
15:06:31.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.322005 30527 external_mini_cluster.cc:1351] Running /tmp/dist-test-task7b0YEV/build/debug/bin/kudu
15:06:31.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task7b0YEV/build/debug/bin/kudu
15:06:31.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-2/wal
15:06:31.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-2/data
15:06:31.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:06:31.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:06:31.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:06:31.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:31.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:31.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:31.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-2/logs
15:06:31.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-2/data/info.pb
15:06:31.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:31.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:06:31.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:06:31.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:06:31.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:06:31.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:31.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:31.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:31.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:06:31.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:06:31.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:06:31.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:06:31.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.207.252:42085
15:06:31.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.207.252
15:06:31.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37191
15:06:31.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.207.212:43549
15:06:31.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:31.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:31.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:31.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:06:31.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.207.254:42611,127.29.207.253:44061,127.29.207.252:42085
15:06:31.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:06:31.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45165/jwks.json
15:06:31.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
15:06:31.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
15:06:31.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:31.426028 31384 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:06:31.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:31.426263 31384 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:06:31.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:31.426340 31384 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:06:31.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:31.429499 31384 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:06:31.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:31.429561 31384 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
15:06:31.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:31.429577 31384 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:06:31.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:31.429594 31384 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:06:31.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:31.429610 31384 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:06:31.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:31.429625 31384 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:06:31.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:31.429638 31384 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45165/jwks.json
15:06:31.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.433359 31384 master_runner.cc:386] Master server non-default flags:
15:06:31.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:31.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.207.212:43549
15:06:31.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:31.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:31.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
15:06:31.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-2/data
15:06:31.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-2/wal
15:06:31.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:06:31.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.207.254:42611,127.29.207.253:44061,127.29.207.252:42085
15:06:31.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
15:06:31.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:31.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:31.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:06:31.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.207.252:42085
15:06:31.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:06:31.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:06:31.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:06:31.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45165/jwks.json
15:06:31.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:31.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:31.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-2/data/info.pb
15:06:31.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.207.252
15:06:31.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37191
15:06:31.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:06:31.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.31384
15:06:31.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:31.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:06:31.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:06:31.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:31.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-2/logs
15:06:31.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:31.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:06:31.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:06:31.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:06:31.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:06:31.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:06:31.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
15:06:31.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Jan 2025 14:43:14 UTC on 24a791456cd2
15:06:31.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4141
15:06:31.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.434419 31384 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:06:31.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.435536 31384 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:06:31.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:31.442179 31389 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:31.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:31.442183 31392 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:31.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:31.442183 31390 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:31.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.442302 31384 server_base.cc:1034] running on GCE node
15:06:31.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.443044 31384 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:06:31.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.443975 31384 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:06:31.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.445153 31384 hybrid_clock.cc:648] HybridClock initialized: now 1736694391445134 us; error 33 us; skew 500 ppm
15:06:31.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.447158 31384 webserver.cc:458] Webserver started at http://127.29.207.252:37191/ using document root <none> and password file <none>
15:06:31.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.447706 31384 fs_manager.cc:362] Metadata directory not provided
15:06:31.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.447769 31384 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:06:31.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.451318 31384 fs_manager.cc:714] Time spent opening directory manager: real 0.002s user 0.003s sys 0.000s
15:06:31.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.453226 31398 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:06:31.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.454279 31384 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
15:06:31.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.454380 31384 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-2/data,/tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-2/wal
15:06:31.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5168bd8e804e45abae06fee65e7c306d"
15:06:31.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:06:28 on dist-test-slave-xnn2"
15:06:31.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.454852 31384 fs_report.cc:389] FS layout report
15:06:31.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:06:31.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-2/wal
15:06:31.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-2/wal
15:06:31.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-2/data/data
15:06:31.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:06:31.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:06:31.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:06:31.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:06:31.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:06:31.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:06:31.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:06:31.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:06:31.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:06:31.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:06:31.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:06:31.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.468724 31384 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:06:31.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.469336 31384 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:06:31.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.469470 31384 kserver.cc:163] Server-wide thread pool size limit: 3276
15:06:31.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.476908 31384 rpc_server.cc:307] RPC server started. Bound to: 127.29.207.252:42085
15:06:31.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.476941 31460 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.207.252:42085 every 8 connection(s)
15:06:31.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.477916 31384 server_base.cc:1166] Dumped server information to /tmp/dist-test-task7b0YEV/test-tmp/mini-kudu-cluster8155963369208076586/master-2/data/info.pb
15:06:31.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.482160 31461 sys_catalog.cc:263] Verifying existing consensus state
15:06:31.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.484090 31461 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d: Bootstrap starting.
15:06:31.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.487305 30527 external_mini_cluster.cc:1413] Started /tmp/dist-test-task7b0YEV/build/debug/bin/kudu as pid 31384
15:06:31.487 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:06:31.487 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.29.207.254:42611
15:06:31.487 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
15:06:31.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.487993 30527 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task7b0YEV/build/debug/bin/kudu with pid 30538
15:06:31.497 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:06:31.497 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.29.207.254:42611(127.29.207.254:42611)] handling channelInactive
15:06:31.497 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.207.254:42611(127.29.207.254:42611)] cleaning up while in state READY due to: connection closed
15:06:31.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.499810 31461 log.cc:826] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d: Log is configured to *not* fsync() on all Append() calls
15:06:31.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.508978 31461 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d: Bootstrap replayed 1/1 log segments. Stats: ops{read=11 overwritten=0 applied=11 ignored=0} inserts{seen=8 ignored=0} mutations{seen=6 ignored=0} orphaned_commits=0. Pending: 0 replicates
15:06:31.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.509413 31461 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d: Bootstrap complete.
15:06:31.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.514215 31461 raft_consensus.cc:357] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a86734036df442d9067f984db9f965e" member_type: VOTER last_known_addr { host: "127.29.207.254" port: 42611 } } peers { permanent_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115" member_type: VOTER last_known_addr { host: "127.29.207.253" port: 44061 } } peers { permanent_uuid: "5168bd8e804e45abae06fee65e7c306d" member_type: VOTER last_known_addr { host: "127.29.207.252" port: 42085 } }
15:06:31.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.514689 31461 raft_consensus.cc:738] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5168bd8e804e45abae06fee65e7c306d, State: Initialized, Role: FOLLOWER
15:06:31.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.515202 31461 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a86734036df442d9067f984db9f965e" member_type: VOTER last_known_addr { host: "127.29.207.254" port: 42611 } } peers { permanent_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115" member_type: VOTER last_known_addr { host: "127.29.207.253" port: 44061 } } peers { permanent_uuid: "5168bd8e804e45abae06fee65e7c306d" member_type: VOTER last_known_addr { host: "127.29.207.252" port: 42085 } }
15:06:31.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.515676 31465 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a86734036df442d9067f984db9f965e" member_type: VOTER last_known_addr { host: "127.29.207.254" port: 42611 } } peers { permanent_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115" member_type: VOTER last_known_addr { host: "127.29.207.253" port: 44061 } } peers { permanent_uuid: "5168bd8e804e45abae06fee65e7c306d" member_type: VOTER last_known_addr { host: "127.29.207.252" port: 42085 } } }
15:06:31.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.515800 31465 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [sys.catalog]: This master's current role is: FOLLOWER
15:06:31.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.516430 31461 sys_catalog.cc:564] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [sys.catalog]: configured and running, proceeding with master startup.
15:06:31.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.516736 31425 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [term 1 FOLLOWER]: Advancing to term 3
15:06:31.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.520004 31478 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "1a86734036df442d9067f984db9f965e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a86734036df442d9067f984db9f965e" member_type: VOTER last_known_addr { host: "127.29.207.254" port: 42611 } } peers { permanent_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115" member_type: VOTER last_known_addr { host: "127.29.207.253" port: 44061 } } peers { permanent_uuid: "5168bd8e804e45abae06fee65e7c306d" member_type: VOTER last_known_addr { host: "127.29.207.252" port: 42085 } } }
15:06:31.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.520133 31478 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [sys.catalog]: This master's current role is: FOLLOWER
15:06:31.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.520467 31465 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1a86734036df442d9067f984db9f965e. Latest consensus state: current_term: 3 leader_uuid: "1a86734036df442d9067f984db9f965e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a86734036df442d9067f984db9f965e" member_type: VOTER last_known_addr { host: "127.29.207.254" port: 42611 } } peers { permanent_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115" member_type: VOTER last_known_addr { host: "127.29.207.253" port: 44061 } } peers { permanent_uuid: "5168bd8e804e45abae06fee65e7c306d" member_type: VOTER last_known_addr { host: "127.29.207.252" port: 42085 } } }
15:06:31.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.520574 31465 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [sys.catalog]: This master's current role is: FOLLOWER
15:06:31.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:31.520731 31399 connection.cc:441] server connection from 127.0.0.1:44680 torn down before Call kudu.consensus.ConsensusService.UpdateConsensus from 127.0.0.1:44680 (request call id 0) could send its response
15:06:31.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.523626 31479 catalog_manager.cc:1260] Loaded cluster ID: 3fab646ada5f4cf2b1a6315d5ebfd421
15:06:31.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.523685 31479 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d: loading cluster ID for follower catalog manager: success
15:06:31.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.522492 31384 master_runner.cc:186] Error getting master registration for 127.29.207.254:42611: Network error: Client connection negotiation failed: client connection to 127.29.207.254:42611: connect: Connection refused (error 111),
15:06:31.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.523954 31384 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:06:31.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.526104 31479 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d: acquiring CA information for follower catalog manager: success
15:06:31.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.527693 31479 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:06:31.600 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.29.207.254:42611(127.29.207.254:42611)] connecting to peer
15:06:31.601 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611
15:06:31.601 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.207.254:42611(127.29.207.254:42611)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611
15:06:31.601 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location master-127.29.207.254:42611(127.29.207.254:42611) for tablet Kudu Master: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611
15:06:31.620 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.29.207.254:42611(127.29.207.254:42611)] connecting to peer
15:06:31.621 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.29.207.252:42085(127.29.207.252:42085)] connecting to peer
15:06:31.621 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611
15:06:31.622 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.29.207.252:42085(127.29.207.252:42085)] Successfully connected to peer
15:06:31.622 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.207.254:42611(127.29.207.254:42611)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611
15:06:31.622 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.207.254:42611: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611
15:06:31.636 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x570f76ed, L:/127.0.0.1:44690 - R:/127.29.207.252:42085] using TOKEN/null
15:06:31.639 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.29.207.254:42611,127.29.207.252:42085,127.29.207.253:44061; will retry
15:06:31.639 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.29.207.254:42611(127.29.207.254:42611)] connecting to peer
15:06:31.640 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611
15:06:31.640 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.207.254:42611(127.29.207.254:42611)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611
15:06:31.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.640956 31125 heartbeater.cc:502] Master 127.29.207.254:42611 was elected leader, sending a full tablet report...
15:06:31.645 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.207.254:42611: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611
15:06:31.646 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.29.207.254:42611,127.29.207.252:42085,127.29.207.253:44061; will retry
15:06:31.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.648383 31124 heartbeater.cc:346] Connected to a master server at 127.29.207.252:42085
15:06:31.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.650043 31415 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "aff4ff4877ed42f58b9a3f6430894003" instance_seqno: 1736694388808955) as {username='slave'} at 127.29.207.194:48461; Asking this server to re-register.
15:06:31.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.650609 31124 heartbeater.cc:463] Registering TS with master...
15:06:31.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.651684 31415 ts_manager.cc:194] Registered new tserver with Master: aff4ff4877ed42f58b9a3f6430894003 (127.29.207.194:36203)
15:06:31.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.29.207.254:42611(127.29.207.254:42611)] connecting to peer
15:06:31.661 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611
15:06:31.662 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.207.254:42611(127.29.207.254:42611)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611
15:06:31.662 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.207.254:42611: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611
15:06:31.663 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.29.207.254:42611,127.29.207.252:42085,127.29.207.253:44061; will retry
15:06:31.680 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.29.207.254:42611(127.29.207.254:42611)] connecting to peer
15:06:31.681 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611
15:06:31.681 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.207.254:42611(127.29.207.254:42611)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611
15:06:31.681 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.207.254:42611: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611
15:06:31.682 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.29.207.254:42611,127.29.207.252:42085,127.29.207.253:44061; will retry
15:06:31.720 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.29.207.254:42611(127.29.207.254:42611)] connecting to peer
15:06:31.722 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611
15:06:31.722 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.207.254:42611(127.29.207.254:42611)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611
15:06:31.722 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.207.254:42611: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611
15:06:31.723 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.29.207.254:42611,127.29.207.252:42085,127.29.207.253:44061; will retry
15:06:31.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.728367 31484 raft_consensus.cc:491] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 1a86734036df442d9067f984db9f965e)
15:06:31.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.728500 31484 raft_consensus.cc:513] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a86734036df442d9067f984db9f965e" member_type: VOTER last_known_addr { host: "127.29.207.254" port: 42611 } } peers { permanent_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115" member_type: VOTER last_known_addr { host: "127.29.207.253" port: 44061 } } peers { permanent_uuid: "5168bd8e804e45abae06fee65e7c306d" member_type: VOTER last_known_addr { host: "127.29.207.252" port: 42085 } }
15:06:31.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.728884 31484 leader_election.cc:290] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 1a86734036df442d9067f984db9f965e (127.29.207.254:42611), 5168bd8e804e45abae06fee65e7c306d (127.29.207.252:42085)
15:06:31.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:31.730158 30637 leader_election.cc:336] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 1a86734036df442d9067f984db9f965e (127.29.207.254:42611): Network error: Client connection negotiation failed: client connection to 127.29.207.254:42611: connect: Connection refused (error 111)
15:06:31.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.732667 31425 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "5168bd8e804e45abae06fee65e7c306d" is_pre_election: true
15:06:31.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.733299 30637 leader_election.cc:304] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 80c9fc0ab13d44ffa2a33e3b8c32f115; no voters: 1a86734036df442d9067f984db9f965e, 5168bd8e804e45abae06fee65e7c306d
15:06:31.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.733487 31484 raft_consensus.cc:2743] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
15:06:31.760 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.29.207.254:42611(127.29.207.254:42611)] connecting to peer
15:06:31.760 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611
15:06:31.761 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.207.254:42611(127.29.207.254:42611)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611
15:06:31.761 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.207.254:42611: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611
15:06:31.761 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.29.207.254:42611,127.29.207.252:42085,127.29.207.253:44061; will retry
15:06:31.762 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] sending RPC to server master-127.29.207.254:42611, [1ms] received response from server master-127.29.207.254:42611: Network error: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.254:42611, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.252:42085, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.253:44061, [22ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.254:42611: Network error: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611, [26ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.253:44061: OK, [38ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.252:42085: OK, [39ms] delaying RPC due to: Service unavailable: Master config (127.29.207.254:42611,127.29.207.252:42085,127.29.207.253:44061) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.254:42611, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.252:42085, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.253:44061, [43ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.252:42085: OK, [43ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.253:44061: OK, [45ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.254:42611: Network error: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611, [46ms] delaying RPC due to: Service unavailable: Master config (127.29.207.254:42611,127.29.207.252:42085,127.29.207.253:44061) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.254:42611, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.252:42085, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.253:44061, [62ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.254:42611: Network error: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611, [62ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.252:42085: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.253:44061: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.29.207.254:42611,127.29.207.252:42085,127.29.207.253:44061) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.254:42611, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.252:42085, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.253:44061, [81ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.254:42611: Network error: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611, [82ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.252:42085: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.253:44061: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.29.207.254:42611,127.29.207.252:42085,127.29.207.253:44061) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.254:42611, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.252:42085, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.253:44061, [122ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.254:42611: Network error: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611, [122ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.252:42085: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.253:44061: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.29.207.254:42611,127.29.207.252:42085,127.29.207.253:44061) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.254:42611, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.252:42085, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.253:44061, [161ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.254:42611: Network error: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611, [161ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.252:42085: OK, [161ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.253:44061: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.29.207.254:42611,127.29.207.252:42085,127.29.207.253:44061) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611, deferred=Deferred@256824056(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1027305234@e4a5a8e -> (continuation of Deferred@1073570089 after retry RPC after error@1131012647) -> (continuation of Deferred@986259872 after retry RPC after error@1712034397) -> (continuation of Deferred@841696156 after retry RPC after error@1046476580) -> (continuation of Deferred@1307051374 after retry RPC after error@2001469428) -> (continuation of Deferred@529823010 after retry RPC after error@1580418851), errback=org.apache.kudu.client.KuduTransaction$$Lambda$84/1543002520@537dfafe -> (continuation of Deferred@1073570089 after retry RPC after error@1131012647) -> (continuation of Deferred@986259872 after retry RPC after error@1712034397) -> (continuation of Deferred@841696156 after retry RPC after error@1046476580) -> (continuation of Deferred@1307051374 after retry RPC after error@2001469428) -> (continuation of Deferred@529823010 after retry RPC after error@1580418851)))
15:06:31.763 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] sending RPC to server master-127.29.207.254:42611, [1ms] received response from server master-127.29.207.254:42611: Network error: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.254:42611, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.252:42085, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.253:44061, [22ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.254:42611: Network error: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611, [26ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.253:44061: OK, [38ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.252:42085: OK, [39ms] delaying RPC due to: Service unavailable: Master config (127.29.207.254:42611,127.29.207.252:42085,127.29.207.253:44061) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.254:42611, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.252:42085, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.253:44061, [43ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.252:42085: OK, [43ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.253:44061: OK, [45ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.254:42611: Network error: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611, [46ms] delaying RPC due to: Service unavailable: Master config (127.29.207.254:42611,127.29.207.252:42085,127.29.207.253:44061) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.254:42611, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.252:42085, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.253:44061, [62ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.254:42611: Network error: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611, [62ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.252:42085: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.253:44061: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.29.207.254:42611,127.29.207.252:42085,127.29.207.253:44061) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.254:42611, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.252:42085, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.253:44061, [81ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.254:42611: Network error: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611, [82ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.252:42085: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.253:44061: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.29.207.254:42611,127.29.207.252:42085,127.29.207.253:44061) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.254:42611, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.252:42085, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.253:44061, [122ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.254:42611: Network error: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611, [122ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.252:42085: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.253:44061: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.29.207.254:42611,127.29.207.252:42085,127.29.207.253:44061) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.254:42611, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.252:42085, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.253:44061, [161ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.254:42611: Network error: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611, [161ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.252:42085: OK, [161ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.253:44061: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.29.207.254:42611,127.29.207.252:42085,127.29.207.253:44061) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611, deferred=Deferred@256824056(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1027305234@e4a5a8e -> (continuation of Deferred@1073570089 after retry RPC after error@1131012647) -> (continuation of Deferred@986259872 after retry RPC after error@1712034397) -> (continuation of Deferred@841696156 after retry RPC after error@1046476580) -> (continuation of Deferred@1307051374 after retry RPC after error@2001469428) -> (continuation of Deferred@529823010 after retry RPC after error@1580418851), errback=org.apache.kudu.client.KuduTransaction$$Lambda$84/1543002520@537dfafe -> (continuation of Deferred@1073570089 after retry RPC after error@1131012647) -> (continuation of Deferred@986259872 after retry RPC after error@1712034397) -> (continuation of Deferred@841696156 after retry RPC after error@1046476580) -> (continuation of Deferred@1307051374 after retry RPC after error@2001469428) -> (continuation of Deferred@529823010 after retry RPC after error@1580418851)))
15:06:31.781 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.29.207.254:42611(127.29.207.254:42611)] connecting to peer
15:06:31.782 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611
15:06:31.782 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.207.254:42611(127.29.207.254:42611)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611
15:06:31.782 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.207.254:42611: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611
15:06:31.783 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.29.207.254:42611,127.29.207.252:42085,127.29.207.253:44061; will retry
15:06:31.799 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.29.207.254:42611(127.29.207.254:42611)] connecting to peer
15:06:31.800 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611
15:06:31.801 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.207.254:42611(127.29.207.254:42611)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611
15:06:31.801 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.207.254:42611: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611
15:06:31.802 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.29.207.254:42611,127.29.207.252:42085,127.29.207.253:44061; will retry
15:06:31.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.819895 31465 raft_consensus.cc:491] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 1a86734036df442d9067f984db9f965e)
15:06:31.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.820098 31465 raft_consensus.cc:513] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a86734036df442d9067f984db9f965e" member_type: VOTER last_known_addr { host: "127.29.207.254" port: 42611 } } peers { permanent_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115" member_type: VOTER last_known_addr { host: "127.29.207.253" port: 44061 } } peers { permanent_uuid: "5168bd8e804e45abae06fee65e7c306d" member_type: VOTER last_known_addr { host: "127.29.207.252" port: 42085 } }
15:06:31.820 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.29.207.254:42611(127.29.207.254:42611)] connecting to peer
15:06:31.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.821533 31465 leader_election.cc:290] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 1a86734036df442d9067f984db9f965e (127.29.207.254:42611), 80c9fc0ab13d44ffa2a33e3b8c32f115 (127.29.207.253:44061)
15:06:31.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:31.822144 31401 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.207.254:42611: connect: Connection refused (error 111)
15:06:31.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:31.823148 31401 leader_election.cc:336] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 1a86734036df442d9067f984db9f965e (127.29.207.254:42611): Network error: Client connection negotiation failed: client connection to 127.29.207.254:42611: connect: Connection refused (error 111)
15:06:31.823 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611
15:06:31.823 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.207.254:42611(127.29.207.254:42611)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611
15:06:31.823 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.207.254:42611: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611
15:06:31.824 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.29.207.254:42611,127.29.207.252:42085,127.29.207.253:44061; will retry
15:06:31.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.826637 30659 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5168bd8e804e45abae06fee65e7c306d" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115" is_pre_election: true
15:06:31.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.826790 30659 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115 [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5168bd8e804e45abae06fee65e7c306d in term 3.
15:06:31.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.827260 31402 leader_election.cc:304] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5168bd8e804e45abae06fee65e7c306d, 80c9fc0ab13d44ffa2a33e3b8c32f115; no voters: 1a86734036df442d9067f984db9f965e
15:06:31.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.827509 31465 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [term 3 FOLLOWER]: Leader pre-election won for term 4
15:06:31.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.827569 31465 raft_consensus.cc:491] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [term 3 FOLLOWER]: Starting leader election (detected failure of leader 1a86734036df442d9067f984db9f965e)
15:06:31.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.827596 31465 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [term 3 FOLLOWER]: Advancing to term 4
15:06:31.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.828395 31465 raft_consensus.cc:513] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a86734036df442d9067f984db9f965e" member_type: VOTER last_known_addr { host: "127.29.207.254" port: 42611 } } peers { permanent_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115" member_type: VOTER last_known_addr { host: "127.29.207.253" port: 44061 } } peers { permanent_uuid: "5168bd8e804e45abae06fee65e7c306d" member_type: VOTER last_known_addr { host: "127.29.207.252" port: 42085 } }
15:06:31.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.828732 31465 leader_election.cc:290] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [CANDIDATE]: Term 4 election: Requested vote from peers 1a86734036df442d9067f984db9f965e (127.29.207.254:42611), 80c9fc0ab13d44ffa2a33e3b8c32f115 (127.29.207.253:44061)
15:06:31.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.829082 30659 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5168bd8e804e45abae06fee65e7c306d" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115"
15:06:31.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.829209 30659 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115 [term 3 FOLLOWER]: Advancing to term 4
15:06:31.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:31.829636 31401 leader_election.cc:336] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer 1a86734036df442d9067f984db9f965e (127.29.207.254:42611): Network error: Client connection negotiation failed: client connection to 127.29.207.254:42611: connect: Connection refused (error 111)
15:06:31.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.830101 30659 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115 [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5168bd8e804e45abae06fee65e7c306d in term 4.
15:06:31.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.830367 31402 leader_election.cc:304] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5168bd8e804e45abae06fee65e7c306d, 80c9fc0ab13d44ffa2a33e3b8c32f115; no voters: 1a86734036df442d9067f984db9f965e
15:06:31.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.830538 31465 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [term 4 FOLLOWER]: Leader election won for term 4
15:06:31.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.830621 31465 raft_consensus.cc:695] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [term 4 LEADER]: Becoming Leader. State: Replica: 5168bd8e804e45abae06fee65e7c306d, State: Running, Role: LEADER
15:06:31.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.830839 31465 consensus_queue.cc:237] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a86734036df442d9067f984db9f965e" member_type: VOTER last_known_addr { host: "127.29.207.254" port: 42611 } } peers { permanent_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115" member_type: VOTER last_known_addr { host: "127.29.207.253" port: 44061 } } peers { permanent_uuid: "5168bd8e804e45abae06fee65e7c306d" member_type: VOTER last_known_addr { host: "127.29.207.252" port: 42085 } }
15:06:31.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.831555 31478 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [sys.catalog]: SysCatalogTable state changed. Reason: New leader 5168bd8e804e45abae06fee65e7c306d. Latest consensus state: current_term: 4 leader_uuid: "5168bd8e804e45abae06fee65e7c306d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a86734036df442d9067f984db9f965e" member_type: VOTER last_known_addr { host: "127.29.207.254" port: 42611 } } peers { permanent_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115" member_type: VOTER last_known_addr { host: "127.29.207.253" port: 44061 } } peers { permanent_uuid: "5168bd8e804e45abae06fee65e7c306d" member_type: VOTER last_known_addr { host: "127.29.207.252" port: 42085 } } }
15:06:31.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.831665 31478 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [sys.catalog]: This master's current role is: LEADER
15:06:31.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.831977 31491 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
15:06:31.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.833379 31491 catalog_manager.cc:670] Loaded metadata for table kudu_system.kudu_transactions [id=49c3be4750994b50904bfce2fa8f396a]
15:06:31.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.833760 31491 catalog_manager.cc:670] Loaded metadata for table txn_manager_fallback_rolling [id=5661f08abeeb4d1483bc1ec2b552a012]
15:06:31.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.834705 31491 tablet_loader.cc:96] loaded metadata for tablet 68b0a288f3184acfb28ff0691b224244 (table txn_manager_fallback_rolling [id=5661f08abeeb4d1483bc1ec2b552a012])
15:06:31.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.834915 31491 tablet_loader.cc:96] loaded metadata for tablet c0712e7c0e614a8a9c9ebb68d8bc2e71 (table txn_manager_fallback_rolling [id=5661f08abeeb4d1483bc1ec2b552a012])
15:06:31.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.835055 31491 tablet_loader.cc:96] loaded metadata for tablet f55bf2b487554de69951e7a3eb3dc955 (table kudu_system.kudu_transactions [id=49c3be4750994b50904bfce2fa8f396a])
15:06:31.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.835141 31491 catalog_manager.cc:1485] Initializing Kudu cluster ID...
15:06:31.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.835482 31491 catalog_manager.cc:1260] Loaded cluster ID: 3fab646ada5f4cf2b1a6315d5ebfd421
15:06:31.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.835523 31491 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
15:06:31.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.836014 31491 catalog_manager.cc:1505] Loading token signing keys...
15:06:31.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.836372 31491 catalog_manager.cc:5910] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d: Loaded TSK: 0
15:06:31.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.836992 31491 catalog_manager.cc:1515] Initializing in-progress tserver states...
15:06:31.840 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.29.207.254:42611(127.29.207.254:42611)] connecting to peer
15:06:31.841 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611
15:06:31.841 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.207.254:42611(127.29.207.254:42611)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611
15:06:31.841 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.207.254:42611: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611
15:06:31.842 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
15:06:31.843 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=4, TimeoutTracker(timeout=250, elapsed=62), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.254:42611, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.252:42085, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.253:44061, [2ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.254:42611: Network error: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611, [2ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.252:42085: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.253:44061: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.29.207.254:42611,127.29.207.252:42085,127.29.207.253:44061) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.254:42611, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.252:42085, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.253:44061, [21ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.254:42611: Network error: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611, [21ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.253:44061: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.252:42085: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.29.207.254:42611,127.29.207.252:42085,127.29.207.253:44061) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.254:42611, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.252:42085, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.253:44061, [43ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.252:42085: OK, [43ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.253:44061: OK, [43ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.254:42611: Network error: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611, [44ms] delaying RPC due to: Service unavailable: Master config (127.29.207.254:42611,127.29.207.252:42085,127.29.207.253:44061) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.254:42611, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.252:42085, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.253:44061, [61ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.254:42611: Network error: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611, [62ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.253:44061: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.252:42085: OK, deferred=Deferred@1291015446(state=PENDING, result=null, callback=(continuation of Deferred@154568781 after retry RPC after error@2062000739) -> (continuation of Deferred@2964622 after retry RPC after error@1453359736) -> (continuation of Deferred@1810435073 after retry RPC after error@752508837), errback=(continuation of Deferred@154568781 after retry RPC after error@2062000739) -> (continuation of Deferred@2964622 after retry RPC after error@1453359736) -> (continuation of Deferred@1810435073 after retry RPC after error@752508837))) after lookup
15:06:31.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.868314 31415 catalog_manager.cc:1909] Servicing CreateTable request from {username='slave'} at 127.0.0.1:44706:
15:06:31.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
15:06:31.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
15:06:31.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:06:31.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
15:06:31.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
15:06:31.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
15:06:31.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
15:06:31.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
15:06:31.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
15:06:31.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
15:06:31.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
15:06:31.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:06:31.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:06:31.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
15:06:31.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
15:06:31.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
15:06:31.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
15:06:31.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
15:06:31.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
15:06:31.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
15:06:31.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
15:06:31.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:06:31.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:06:31.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
15:06:31.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
15:06:31.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
15:06:31.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
15:06:31.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
15:06:31.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
15:06:31.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
15:06:31.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
15:06:31.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:06:31.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:06:31.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
15:06:31.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
15:06:31.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
15:06:31.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
15:06:31.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
15:06:31.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
15:06:31.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
15:06:31.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
15:06:31.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:06:31.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:06:31.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
15:06:31.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
15:06:31.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
15:06:31.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
15:06:31.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:06:31.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
15:06:31.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
15:06:31.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:06:31.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
15:06:31.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:06:31.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:06:31.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:06:31.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
15:06:31.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:31.870823 31461 master.cc:440] Invalid argument: unable to initialize TxnManager: Error creating table kudu_system.kudu_transactions on the master: not enough live tablet servers to create a table with the requested replication factor 3; 1 tablet servers are alive: unable to init TxnManager, will retry in 1.000s
15:06:31.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:31.912245 31401 consensus_peers.cc:487] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d -> Peer 1a86734036df442d9067f984db9f965e (127.29.207.254:42611): Couldn't send request to peer 1a86734036df442d9067f984db9f965e. Status: Network error: Client connection negotiation failed: client connection to 127.29.207.254:42611: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
15:06:31.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.917618 30659 raft_consensus.cc:1270] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115 [term 4 FOLLOWER]: Refusing update from remote peer 5168bd8e804e45abae06fee65e7c306d: Log matching property violated. Preceding OpId in replica: term: 3 index: 12. Preceding OpId from leader: term: 4 index: 13. (index mismatch)
15:06:31.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.918147 31478 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [LEADER]: Connected to new peer: Peer: permanent_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115" member_type: VOTER last_known_addr { host: "127.29.207.253" port: 44061 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
15:06:31.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.919374 31484 sys_catalog.cc:455] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 5168bd8e804e45abae06fee65e7c306d. Latest consensus state: current_term: 4 leader_uuid: "5168bd8e804e45abae06fee65e7c306d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a86734036df442d9067f984db9f965e" member_type: VOTER last_known_addr { host: "127.29.207.254" port: 42611 } } peers { permanent_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115" member_type: VOTER last_known_addr { host: "127.29.207.253" port: 44061 } } peers { permanent_uuid: "5168bd8e804e45abae06fee65e7c306d" member_type: VOTER last_known_addr { host: "127.29.207.252" port: 42085 } } }
15:06:31.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.919504 31484 sys_catalog.cc:458] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115 [sys.catalog]: This master's current role is: FOLLOWER
15:06:31.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.920414 31465 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 4 leader_uuid: "5168bd8e804e45abae06fee65e7c306d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a86734036df442d9067f984db9f965e" member_type: VOTER last_known_addr { host: "127.29.207.254" port: 42611 } } peers { permanent_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115" member_type: VOTER last_known_addr { host: "127.29.207.253" port: 44061 } } peers { permanent_uuid: "5168bd8e804e45abae06fee65e7c306d" member_type: VOTER last_known_addr { host: "127.29.207.252" port: 42085 } } }
15:06:31.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.920529 31465 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d [sys.catalog]: This master's current role is: LEADER
15:06:31.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.920877 31484 sys_catalog.cc:455] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "5168bd8e804e45abae06fee65e7c306d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a86734036df442d9067f984db9f965e" member_type: VOTER last_known_addr { host: "127.29.207.254" port: 42611 } } peers { permanent_uuid: "80c9fc0ab13d44ffa2a33e3b8c32f115" member_type: VOTER last_known_addr { host: "127.29.207.253" port: 44061 } } peers { permanent_uuid: "5168bd8e804e45abae06fee65e7c306d" member_type: VOTER last_known_addr { host: "127.29.207.252" port: 42085 } } }
15:06:31.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:31.921000 31484 sys_catalog.cc:458] T 00000000000000000000000000000000 P 80c9fc0ab13d44ffa2a33e3b8c32f115 [sys.catalog]: This master's current role is: FOLLOWER
15:06:32.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:32.032052 31445 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:44690 (request call id 10) took 188 ms (client timeout 188 ms). Trace:
15:06:32.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:32.032171 31445 rpcz_store.cc:269] 0112 15:06:31.843570 (+ 0us) service_pool.cc:168] Inserting onto call queue
15:06:32.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:06:31.843634 (+ 64us) service_pool.cc:225] Handling call
15:06:32.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:06:32.032011 (+188377us) inbound_call.cc:173] Queueing success response
15:06:32.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"dns_us":375,"init.queue_time_us":47,"thread_start_us":268,"threads_started":4}
15:06:32.034 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=5, TimeoutTracker(timeout=250, elapsed=252), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.254:42611, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.252:42085, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.253:44061, [2ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.254:42611: Network error: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611, [2ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.252:42085: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.253:44061: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.29.207.254:42611,127.29.207.252:42085,127.29.207.253:44061) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.254:42611, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.252:42085, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.253:44061, [21ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.254:42611: Network error: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611, [21ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.253:44061: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.252:42085: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.29.207.254:42611,127.29.207.252:42085,127.29.207.253:44061) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.254:42611, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.252:42085, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.253:44061, [43ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.252:42085: OK, [43ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.253:44061: OK, [43ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.254:42611: Network error: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611, [44ms] delaying RPC due to: Service unavailable: Master config (127.29.207.254:42611,127.29.207.252:42085,127.29.207.253:44061) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.254:42611, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.252:42085, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.253:44061, [61ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.254:42611: Network error: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611, [62ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.253:44061: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.252:42085: OK, [63ms] sending RPC to server master-127.29.207.252:42085, [253ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@1291015446(state=PENDING, result=null, callback=(continuation of Deferred@154568781 after retry RPC after error@2062000739) -> (continuation of Deferred@2964622 after retry RPC after error@1453359736) -> (continuation of Deferred@1810435073 after retry RPC after error@752508837) -> (continuation of Deferred@1165680636 after retry RPC@387655273), errback=(continuation of Deferred@154568781 after retry RPC after error@2062000739) -> (continuation of Deferred@2964622 after retry RPC after error@1453359736) -> (continuation of Deferred@1810435073 after retry RPC after error@752508837) -> (continuation of Deferred@1165680636 after retry RPC@387655273)))
15:06:32.034 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=5, TimeoutTracker(timeout=250, elapsed=252), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.254:42611, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.252:42085, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.253:44061, [2ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.254:42611: Network error: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611, [2ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.252:42085: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.253:44061: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.29.207.254:42611,127.29.207.252:42085,127.29.207.253:44061) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.254:42611, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.252:42085, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.253:44061, [21ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.254:42611: Network error: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611, [21ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.253:44061: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.252:42085: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.29.207.254:42611,127.29.207.252:42085,127.29.207.253:44061) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.254:42611, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.252:42085, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.253:44061, [43ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.252:42085: OK, [43ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.253:44061: OK, [43ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.254:42611: Network error: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611, [44ms] delaying RPC due to: Service unavailable: Master config (127.29.207.254:42611,127.29.207.252:42085,127.29.207.253:44061) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.254:42611, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.252:42085, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.207.253:44061, [61ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.254:42611: Network error: Failed to connect to peer master-127.29.207.254:42611(127.29.207.254:42611): Connection refused: /127.29.207.254:42611, [62ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.253:44061: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.29.207.252:42085: OK, [63ms] sending RPC to server master-127.29.207.252:42085, [253ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@1291015446(state=PENDING, result=null, callback=(continuation of Deferred@154568781 after retry RPC after error@2062000739) -> (continuation of Deferred@2964622 after retry RPC after error@1453359736) -> (continuation of Deferred@1810435073 after retry RPC after error@752508837) -> (continuation of Deferred@1165680636 after retry RPC@387655273), errback=(continuation of Deferred@154568781 after retry RPC after error@2062000739) -> (continuation of Deferred@2964622 after retry RPC after error@1453359736) -> (continuation of Deferred@1810435073 after retry RPC after error@752508837) -> (continuation of Deferred@1165680636 after retry RPC@387655273)))
15:06:32.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:32.061656 31267 heartbeater.cc:346] Connected to a master server at 127.29.207.252:42085
15:06:32.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:32.062705 31415 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "57641dcf709949098d39f0e8b0fbe6a7" instance_seqno: 1736694388991402) as {username='slave'} at 127.29.207.195:32885; Asking this server to re-register.
15:06:32.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:32.063153 31267 heartbeater.cc:463] Registering TS with master...
15:06:32.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:32.063344 31267 heartbeater.cc:510] Master 127.29.207.252:42085 requested a full tablet report, sending...
15:06:32.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:32.064237 31415 ts_manager.cc:194] Registered new tserver with Master: 57641dcf709949098d39f0e8b0fbe6a7 (127.29.207.195:40677)
15:06:32.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:32.091758 30982 heartbeater.cc:346] Connected to a master server at 127.29.207.252:42085
15:06:32.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:32.092783 31415 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "e6001a1b382847169cac813c03722887" instance_seqno: 1736694388629863) as {username='slave'} at 127.29.207.193:53703; Asking this server to re-register.
15:06:32.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:32.093200 30982 heartbeater.cc:463] Registering TS with master...
15:06:32.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:32.093382 30982 heartbeater.cc:510] Master 127.29.207.252:42085 requested a full tablet report, sending...
15:06:32.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:32.094201 31415 ts_manager.cc:194] Registered new tserver with Master: e6001a1b382847169cac813c03722887 (127.29.207.193:39647)
15:06:32.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:32.131520 30866 txn_status_manager.cc:1391] automatically aborted stale txn (ID 0) past 1.044s from last keepalive heartbeat (effective timeout is 1.000s)
15:06:32.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:32.140400 31506 tablet_replica.cc:1307] Aborted: operation has been aborted: cancelling pending write operations
15:06:32.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:32.291913 31445 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:44690 (request call id 11) took 250 ms (client timeout 250 ms). Trace:
15:06:32.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:32.292025 31445 rpcz_store.cc:269] 0112 15:06:32.041639 (+ 0us) service_pool.cc:168] Inserting onto call queue
15:06:32.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:06:32.041713 (+ 74us) service_pool.cc:225] Handling call
15:06:32.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:06:32.291882 (+250169us) inbound_call.cc:173] Queueing success response
15:06:32.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
15:06:32.293 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=252), Traces: [0ms] sending RPC to server master-127.29.207.252:42085, [251ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@875673999(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1027305234@4ae8efdb, errback=org.apache.kudu.client.KuduTransaction$$Lambda$84/1543002520@59fe4c45))
15:06:32.293 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=252), Traces: [0ms] sending RPC to server master-127.29.207.252:42085, [251ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@875673999(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1027305234@4ae8efdb, errback=org.apache.kudu.client.KuduTransaction$$Lambda$84/1543002520@59fe4c45))
15:06:32.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:32.441963 31401 consensus_peers.cc:487] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d -> Peer 1a86734036df442d9067f984db9f965e (127.29.207.254:42611): Couldn't send request to peer 1a86734036df442d9067f984db9f965e. Status: Network error: Client connection negotiation failed: client connection to 127.29.207.254:42611: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
15:06:32.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:32.550937 31445 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:44690 (request call id 12) took 250 ms (client timeout 250 ms). Trace:
15:06:32.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:32.551038 31445 rpcz_store.cc:269] 0112 15:06:32.300753 (+ 0us) service_pool.cc:168] Inserting onto call queue
15:06:32.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:06:32.300827 (+ 74us) service_pool.cc:225] Handling call
15:06:32.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:06:32.550901 (+250074us) inbound_call.cc:173] Queueing success response
15:06:32.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
15:06:32.552 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.29.207.252:42085, [251ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@395837821(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1027305234@2edc3fc0, errback=org.apache.kudu.client.KuduTransaction$$Lambda$84/1543002520@23c78bf7))
15:06:32.552 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.29.207.252:42085, [251ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@395837821(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1027305234@2edc3fc0, errback=org.apache.kudu.client.KuduTransaction$$Lambda$84/1543002520@23c78bf7))
15:06:32.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:32.811100 31445 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:44690 (request call id 13) took 250 ms (client timeout 250 ms). Trace:
15:06:32.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:32.811199 31445 rpcz_store.cc:269] 0112 15:06:32.560939 (+ 0us) service_pool.cc:168] Inserting onto call queue
15:06:32.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:06:32.561005 (+ 66us) service_pool.cc:225] Handling call
15:06:32.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:06:32.811064 (+250059us) inbound_call.cc:173] Queueing success response
15:06:32.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
15:06:32.812 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.29.207.252:42085, [251ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@149977006(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1027305234@5fb41c24, errback=org.apache.kudu.client.KuduTransaction$$Lambda$84/1543002520@10d90a2d))
15:06:32.812 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.29.207.252:42085, [251ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@149977006(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1027305234@5fb41c24, errback=org.apache.kudu.client.KuduTransaction$$Lambda$84/1543002520@10d90a2d))
15:06:32.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:32.888006 31415 catalog_manager.cc:1909] Servicing CreateTable request from {username='slave'} at 127.0.0.1:44718:
15:06:32.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
15:06:32.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
15:06:32.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:06:32.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
15:06:32.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
15:06:32.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
15:06:32.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
15:06:32.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
15:06:32.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
15:06:32.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
15:06:32.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
15:06:32.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:06:32.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:06:32.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
15:06:32.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
15:06:32.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
15:06:32.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
15:06:32.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
15:06:32.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
15:06:32.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
15:06:32.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
15:06:32.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:06:32.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:06:32.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
15:06:32.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
15:06:32.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
15:06:32.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
15:06:32.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
15:06:32.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
15:06:32.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
15:06:32.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
15:06:32.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:06:32.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:06:32.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
15:06:32.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
15:06:32.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
15:06:32.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
15:06:32.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
15:06:32.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
15:06:32.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
15:06:32.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
15:06:32.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:06:32.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:06:32.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
15:06:32.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
15:06:32.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
15:06:32.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
15:06:32.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:06:32.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
15:06:32.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
15:06:32.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:06:32.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
15:06:32.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:06:32.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:06:32.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:06:32.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
15:06:32.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:32.888626 31415 catalog_manager.cc:6885] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
15:06:32.901 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) terminating keepalive task (txn ID 0) due to exception org.apache.kudu.client.NonRecoverableException: Failed to write to server: e6001a1b382847169cac813c03722887 (127.29.207.193:39647): transaction ID 0 is not available for further commits: state: ABORTED user: "slave" start_timestamp: 1736694389 last_transition_timestamp: 1736694392
15:06:32.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:32.967314 31401 consensus_peers.cc:487] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d -> Peer 1a86734036df442d9067f984db9f965e (127.29.207.254:42611): Couldn't send request to peer 1a86734036df442d9067f984db9f965e. Status: Network error: Client connection negotiation failed: client connection to 127.29.207.254:42611: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
15:06:33.501 [DEBUG - main] (Connection.java:575) [peer master-127.29.207.253:44061(127.29.207.253:44061)] disconnecting while in state READY
15:06:33.501 [DEBUG - main] (Connection.java:575) [peer 57641dcf709949098d39f0e8b0fbe6a7(127.29.207.195:40677)] disconnecting while in state READY
15:06:33.501 [DEBUG - main] (Connection.java:575) [peer master-127.29.207.252:42085(127.29.207.252:42085)] disconnecting while in state READY
15:06:33.501 [DEBUG - main] (Connection.java:575) [peer master-127.29.207.254:42611(127.29.207.254:42611)] disconnecting while in state TERMINATED
15:06:33.502 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
15:06:33.502 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.29.207.253:44061(127.29.207.253:44061)] handling channelInactive
15:06:33.502 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.207.253:44061(127.29.207.253:44061)] cleaning up while in state READY due to: connection closed
15:06:33.502 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.29.207.252:42085(127.29.207.252:42085)] handling channelInactive
15:06:33.502 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.207.252:42085(127.29.207.252:42085)] cleaning up while in state READY due to: connection closed
15:06:33.502 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 57641dcf709949098d39f0e8b0fbe6a7(127.29.207.195:40677)] handling channelInactive
15:06:33.502 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 57641dcf709949098d39f0e8b0fbe6a7(127.29.207.195:40677)] cleaning up while in state READY due to: connection closed
15:06:33.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:33.502605 30527 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task7b0YEV/build/debug/bin/kudu with pid 30845
15:06:33.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:33.511409 30527 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task7b0YEV/build/debug/bin/kudu with pid 30987
15:06:33.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:33.517285 30527 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task7b0YEV/build/debug/bin/kudu with pid 31129
15:06:33.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:33.523000 30527 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task7b0YEV/build/debug/bin/kudu with pid 30617
15:06:33.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:06:33.526245 31401 consensus_peers.cc:487] T 00000000000000000000000000000000 P 5168bd8e804e45abae06fee65e7c306d -> Peer 1a86734036df442d9067f984db9f965e (127.29.207.254:42611): Couldn't send request to peer 1a86734036df442d9067f984db9f965e. Status: Network error: Client connection negotiation failed: client connection to 127.29.207.254:42611: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
15:06:33.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:06:33.529318 30527 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task7b0YEV/build/debug/bin/kudu with pid 31384
15:06:33.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-12T15:06:33Z chronyd exiting
15:06:33.582 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager: failed attempt 1
java.lang.AssertionError: Aborted: transaction was aborted
at org.junit.Assert.fail(Assert.java:89) ~[junit-4.13.2.jar:4.13.2]
at org.junit.Assert.assertTrue(Assert.java:42) ~[junit-4.13.2.jar:4.13.2]
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
java.lang.AssertionError: Aborted: transaction was aborted
at org.junit.Assert.fail(Assert.java:89)
at org.junit.Assert.assertTrue(Assert.java:42)
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)