15:02:49.390 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
15:02:49.390 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskDlDeJ5/test-tmp
15:02:49.391 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/asan/bin
15:02:49.391 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskDlDeJ5/java/build/dist-test/../../../build/asan/bin/kudu, test, mini_cluster, --serialization=pb]
15:02:49.392 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119"
extra_master_flags: "--txn_manager_enabled"
extra_master_flags: "--raft_heartbeat_interval_ms=100"
extra_tserver_flags: "--txn_keepalive_interval_ms=1000"
extra_tserver_flags: "--txn_staleness_tracker_interval_ms=250"
extra_tserver_flags: "--enable_txn_system_client_init=true"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
15:02:49.698 [DEBUG - main] (MiniKuduCluster.java:195) Response:
15:02:49.698 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
15:02:49.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:49.700675 20176 mini_oidc.cc:150] Starting JWKS server
15:02:49.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:49.703035 20176 webserver.cc:458] Webserver started at http://127.0.0.1:39893/ using document root <none> and password file <none>
15:02:49.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:49.703239 20176 mini_oidc.cc:186] Starting OIDC Discovery server
15:02:49.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:49.703794 20176 webserver.cc:458] Webserver started at http://127.0.0.1:40879/ using document root <none> and password file <none>
15:02:49.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-12T15:02:49Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
15:02:49.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-12T15:02:49Z Disabled control of system clock
15:02:49.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:49.731892 20176 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskDlDeJ5/build/asan/bin/kudu
15:02:49.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDlDeJ5/build/asan/bin/kudu
15:02:49.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/master-0/wal
15:02:49.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/master-0/data
15:02:49.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:02:49.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:02:49.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:02:49.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:02:49.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:02:49.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:02:49.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/master-0/logs
15:02:49.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/master-0/data/info.pb
15:02:49.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:02:49.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:02:49.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:02:49.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:02:49.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:02:49.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:02:49.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:02:49.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:02:49.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:02:49.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:02:49.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:02:49.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:02:49.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.180.62:46081
15:02:49.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.180.62
15:02:49.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:02:49.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.180.20:41217
15:02:49.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:02:49.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:02:49.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:02:49.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:02:49.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797
15:02:49.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:02:49.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39893/jwks.json
15:02:49.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
15:02:49.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
15:02:50.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:50.034379 20187 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:02:50.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:50.034667 20187 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:02:50.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:50.034761 20187 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:02:50.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:50.042274 20187 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:02:50.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:50.042380 20187 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
15:02:50.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:50.042418 20187 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:02:50.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:50.042466 20187 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:02:50.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:50.042512 20187 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:02:50.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:50.042570 20187 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:02:50.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:50.042610 20187 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39893/jwks.json
15:02:50.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.051949 20187 master_runner.cc:386] Master server non-default flags:
15:02:50.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:02:50.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.180.20:41217
15:02:50.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:02:50.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:02:50.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
15:02:50.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/master-0/data
15:02:50.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/master-0/wal
15:02:50.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:02:50.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797
15:02:50.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
15:02:50.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:02:50.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:02:50.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:02:50.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.180.62:46081
15:02:50.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:02:50.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:02:50.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:02:50.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39893/jwks.json
15:02:50.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:02:50.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:02:50.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/master-0/data/info.pb
15:02:50.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.180.62
15:02:50.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:02:50.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:02:50.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:02:50.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:02:50.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:02:50.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:02:50.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/master-0/logs
15:02:50.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:02:50.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:02:50.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:02:50.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:02:50.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:02:50.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:02:50.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:02:50.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Jan 2025 14:43:22 UTC on 24a791456cd2
15:02:50.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4143
15:02:50.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
15:02:50.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.053377 20187 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:02:50.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.054987 20187 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:02:50.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:50.064963 20195 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:02:50.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:50.065367 20193 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:02:50.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:50.065572 20192 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:02:50.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.066279 20187 server_base.cc:1034] running on GCE node
15:02:50.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.066988 20187 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:02:50.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.068262 20187 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:02:50.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.069464 20187 hybrid_clock.cc:648] HybridClock initialized: now 1736694170069429 us; error 26 us; skew 500 ppm
15:02:50.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.069806 20187 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:02:50.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.071864 20187 webserver.cc:458] Webserver started at http://127.19.180.62:37911/ using document root <none> and password file <none>
15:02:50.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.072535 20187 fs_manager.cc:362] Metadata directory not provided
15:02:50.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.072634 20187 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:02:50.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.072968 20187 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:02:50.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.075429 20187 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/master-0/data/instance:
15:02:50.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3ea36a1a750c40b98e9e225442fadbd1"
15:02:50.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:02:50 on dist-test-slave-b45r"
15:02:50.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.076201 20187 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/master-0/wal/instance:
15:02:50.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3ea36a1a750c40b98e9e225442fadbd1"
15:02:50.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:02:50 on dist-test-slave-b45r"
15:02:50.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.080857 20187 fs_manager.cc:696] Time spent creating directory manager: real 0.004s user 0.005s sys 0.000s
15:02:50.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.083878 20201 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:02:50.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.085024 20187 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.003s sys 0.000s
15:02:50.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.085177 20187 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/master-0/data,/tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/master-0/wal
15:02:50.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3ea36a1a750c40b98e9e225442fadbd1"
15:02:50.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:02:50 on dist-test-slave-b45r"
15:02:50.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.085361 20187 fs_report.cc:389] FS layout report
15:02:50.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:02:50.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/master-0/wal
15:02:50.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/master-0/wal
15:02:50.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/master-0/data/data
15:02:50.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:02:50.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:02:50.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:02:50.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:02:50.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:02:50.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:02:50.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:02:50.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:02:50.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:02:50.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:02:50.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:02:50.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.106721 20187 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:02:50.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.107540 20187 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:02:50.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.107827 20187 kserver.cc:163] Server-wide thread pool size limit: 3276
15:02:50.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.126536 20187 rpc_server.cc:307] RPC server started. Bound to: 127.19.180.62:46081
15:02:50.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.126565 20262 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.180.62:46081 every 8 connection(s)
15:02:50.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.127897 20187 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/master-0/data/info.pb
15:02:50.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.131332 20263 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:02:50.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.133584 20176 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskDlDeJ5/build/asan/bin/kudu as pid 20187
15:02:50.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.133850 20176 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/master-0/wal/instance
15:02:50.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.136581 20176 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskDlDeJ5/build/asan/bin/kudu
15:02:50.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDlDeJ5/build/asan/bin/kudu
15:02:50.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/master-1/wal
15:02:50.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/master-1/data
15:02:50.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:02:50.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:02:50.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:02:50.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:02:50.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:02:50.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:02:50.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/master-1/logs
15:02:50.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/master-1/data/info.pb
15:02:50.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:02:50.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:02:50.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:02:50.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:02:50.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:02:50.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:02:50.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:02:50.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:02:50.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:02:50.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:02:50.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:02:50.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:02:50.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.180.61:37725
15:02:50.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.180.61
15:02:50.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:02:50.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.180.20:41217
15:02:50.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:02:50.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:02:50.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:02:50.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:02:50.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797
15:02:50.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:02:50.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39893/jwks.json
15:02:50.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
15:02:50.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
15:02:50.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.138798 20263 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.180.62" port: 46081 } has no permanent_uuid. Determining permanent_uuid...
15:02:50.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.154754 20263 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.180.61" port: 37725 } has no permanent_uuid. Determining permanent_uuid...
15:02:50.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:50.155751 20203 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.180.61:37725: connect: Connection refused (error 111)
15:02:50.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:50.157109 20263 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.19.180.61:37725: Network error: Client connection negotiation failed: client connection to 127.19.180.61:37725: connect: Connection refused (error 111)
15:02:50.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.206398 20263 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.180.61" port: 37725 } attempt: 1
15:02:50.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:50.208407 20263 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.19.180.61:37725: Network error: Client connection negotiation failed: client connection to 127.19.180.61:37725: connect: Connection refused (error 111)
15:02:50.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.276649 20263 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.180.61" port: 37725 } attempt: 2
15:02:50.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:50.278492 20263 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.19.180.61:37725: Network error: Client connection negotiation failed: client connection to 127.19.180.61:37725: connect: Connection refused (error 111)
15:02:50.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.369750 20263 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.180.61" port: 37725 } attempt: 3
15:02:50.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:50.371660 20263 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.19.180.61:37725: Network error: Client connection negotiation failed: client connection to 127.19.180.61:37725: connect: Connection refused (error 111)
15:02:50.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:50.447906 20265 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:02:50.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:50.448175 20265 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:02:50.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:50.448271 20265 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:02:50.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:50.455735 20265 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:02:50.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:50.455838 20265 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
15:02:50.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:50.455875 20265 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:02:50.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:50.455924 20265 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:02:50.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:50.455961 20265 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:02:50.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:50.456012 20265 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:02:50.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:50.456043 20265 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39893/jwks.json
15:02:50.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.465133 20265 master_runner.cc:386] Master server non-default flags:
15:02:50.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:02:50.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.180.20:41217
15:02:50.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:02:50.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:02:50.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
15:02:50.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/master-1/data
15:02:50.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/master-1/wal
15:02:50.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:02:50.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797
15:02:50.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
15:02:50.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:02:50.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:02:50.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:02:50.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.180.61:37725
15:02:50.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:02:50.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:02:50.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:02:50.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39893/jwks.json
15:02:50.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:02:50.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:02:50.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/master-1/data/info.pb
15:02:50.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.180.61
15:02:50.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:02:50.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:02:50.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:02:50.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:02:50.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:02:50.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:02:50.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/master-1/logs
15:02:50.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:02:50.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:02:50.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:02:50.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:02:50.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:02:50.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:02:50.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:02:50.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Jan 2025 14:43:22 UTC on 24a791456cd2
15:02:50.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4143
15:02:50.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
15:02:50.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.466607 20265 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:02:50.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.468242 20265 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:02:50.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:50.478029 20274 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:02:50.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:50.478479 20276 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:02:50.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:50.478689 20273 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:02:50.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.479159 20265 server_base.cc:1034] running on GCE node
15:02:50.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.479892 20265 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:02:50.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.481226 20265 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:02:50.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.482426 20265 hybrid_clock.cc:648] HybridClock initialized: now 1736694170482382 us; error 27 us; skew 500 ppm
15:02:50.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.482767 20265 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:02:50.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.484895 20265 webserver.cc:458] Webserver started at http://127.19.180.61:41737/ using document root <none> and password file <none>
15:02:50.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.485590 20265 fs_manager.cc:362] Metadata directory not provided
15:02:50.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.485692 20265 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:02:50.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.485985 20265 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:02:50.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.488474 20265 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/master-1/data/instance:
15:02:50.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ff11b88709b14cda87f5cecd80a792d6"
15:02:50.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:02:50 on dist-test-slave-b45r"
15:02:50.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.489264 20265 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/master-1/wal/instance:
15:02:50.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ff11b88709b14cda87f5cecd80a792d6"
15:02:50.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:02:50 on dist-test-slave-b45r"
15:02:50.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.494043 20265 fs_manager.cc:696] Time spent creating directory manager: real 0.004s user 0.002s sys 0.004s
15:02:50.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.497192 20282 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:02:50.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.498346 20265 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.000s sys 0.002s
15:02:50.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.498510 20265 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/master-1/data,/tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/master-1/wal
15:02:50.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ff11b88709b14cda87f5cecd80a792d6"
15:02:50.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:02:50 on dist-test-slave-b45r"
15:02:50.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.498697 20265 fs_report.cc:389] FS layout report
15:02:50.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:02:50.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/master-1/wal
15:02:50.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/master-1/wal
15:02:50.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/master-1/data/data
15:02:50.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:02:50.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:02:50.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:02:50.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:02:50.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:02:50.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:02:50.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:02:50.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:02:50.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:02:50.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:02:50.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:02:50.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.514858 20263 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.180.61" port: 37725 } attempt: 4
15:02:50.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:50.516476 20263 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.19.180.61:37725: Network error: Client connection negotiation failed: client connection to 127.19.180.61:37725: connect: Connection refused (error 111)
15:02:50.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.530022 20265 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:02:50.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.530884 20265 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:02:50.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.531194 20265 kserver.cc:163] Server-wide thread pool size limit: 3276
15:02:50.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.549983 20265 rpc_server.cc:307] RPC server started. Bound to: 127.19.180.61:37725
15:02:50.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.550014 20343 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.180.61:37725 every 8 connection(s)
15:02:50.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.551319 20265 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/master-1/data/info.pb
15:02:50.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.554283 20176 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskDlDeJ5/build/asan/bin/kudu as pid 20265
15:02:50.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.554461 20176 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/master-1/wal/instance
15:02:50.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.554780 20344 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:02:50.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.555517 20176 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskDlDeJ5/build/asan/bin/kudu
15:02:50.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDlDeJ5/build/asan/bin/kudu
15:02:50.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/master-2/wal
15:02:50.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/master-2/data
15:02:50.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:02:50.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:02:50.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:02:50.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:02:50.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:02:50.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:02:50.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/master-2/logs
15:02:50.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/master-2/data/info.pb
15:02:50.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:02:50.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:02:50.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:02:50.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:02:50.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:02:50.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:02:50.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:02:50.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:02:50.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:02:50.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:02:50.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:02:50.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:02:50.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.180.60:32797
15:02:50.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.180.60
15:02:50.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:02:50.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.180.20:41217
15:02:50.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:02:50.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:02:50.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:02:50.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:02:50.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797
15:02:50.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:02:50.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39893/jwks.json
15:02:50.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
15:02:50.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
15:02:50.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.560441 20344 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.180.62" port: 46081 } has no permanent_uuid. Determining permanent_uuid...
15:02:50.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.573400 20344 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.180.61" port: 37725 } has no permanent_uuid. Determining permanent_uuid...
15:02:50.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.582042 20344 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.180.60" port: 32797 } has no permanent_uuid. Determining permanent_uuid...
15:02:50.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:50.582921 20286 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.180.60:32797: connect: Connection refused (error 111)
15:02:50.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:50.584412 20344 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.19.180.60:32797: Network error: Client connection negotiation failed: client connection to 127.19.180.60:32797: connect: Connection refused (error 111)
15:02:50.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.633672 20344 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.180.60" port: 32797 } attempt: 1
15:02:50.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:50.635529 20344 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.19.180.60:32797: Network error: Client connection negotiation failed: client connection to 127.19.180.60:32797: connect: Connection refused (error 111)
15:02:50.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.703722 20344 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.180.60" port: 32797 } attempt: 2
15:02:50.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:50.705471 20344 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.19.180.60:32797: Network error: Client connection negotiation failed: client connection to 127.19.180.60:32797: connect: Connection refused (error 111)
15:02:50.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.796674 20344 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.180.60" port: 32797 } attempt: 3
15:02:50.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:50.798507 20344 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.19.180.60:32797: Network error: Client connection negotiation failed: client connection to 127.19.180.60:32797: connect: Connection refused (error 111)
15:02:50.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.815711 20263 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.180.61" port: 37725 } attempt: 5
15:02:50.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.822016 20263 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.180.60" port: 32797 } has no permanent_uuid. Determining permanent_uuid...
15:02:50.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:50.823661 20263 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.19.180.60:32797: Network error: Client connection negotiation failed: client connection to 127.19.180.60:32797: connect: Connection refused (error 111)
15:02:50.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:50.863444 20346 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:02:50.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:50.863721 20346 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:02:50.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:50.863843 20346 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:02:50.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:50.871147 20346 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:02:50.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:50.871248 20346 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
15:02:50.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:50.871295 20346 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:02:50.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:50.871361 20346 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:02:50.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:50.871413 20346 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:02:50.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:50.871484 20346 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:02:50.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:50.871542 20346 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39893/jwks.json
15:02:50.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.874843 20263 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.180.60" port: 32797 } attempt: 1
15:02:50.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:50.876504 20263 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.19.180.60:32797: Network error: Client connection negotiation failed: client connection to 127.19.180.60:32797: connect: Connection refused (error 111)
15:02:50.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.880577 20346 master_runner.cc:386] Master server non-default flags:
15:02:50.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:02:50.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.180.20:41217
15:02:50.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:02:50.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:02:50.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
15:02:50.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/master-2/data
15:02:50.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/master-2/wal
15:02:50.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:02:50.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797
15:02:50.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
15:02:50.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:02:50.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:02:50.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:02:50.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.180.60:32797
15:02:50.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:02:50.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:02:50.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:02:50.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39893/jwks.json
15:02:50.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:02:50.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:02:50.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/master-2/data/info.pb
15:02:50.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.180.60
15:02:50.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:02:50.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:02:50.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:02:50.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:02:50.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:02:50.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:02:50.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/master-2/logs
15:02:50.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:02:50.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:02:50.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:02:50.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:02:50.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:02:50.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:02:50.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:02:50.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Jan 2025 14:43:22 UTC on 24a791456cd2
15:02:50.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4143
15:02:50.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
15:02:50.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.882324 20346 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:02:50.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.883970 20346 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:02:50.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:50.893457 20354 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:02:50.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:50.893901 20355 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:02:50.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:50.894248 20357 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:02:50.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.894523 20346 server_base.cc:1034] running on GCE node
15:02:50.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.895255 20346 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:02:50.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.896517 20346 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:02:50.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.897706 20346 hybrid_clock.cc:648] HybridClock initialized: now 1736694170897663 us; error 32 us; skew 500 ppm
15:02:50.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.898029 20346 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:02:50.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.900060 20346 webserver.cc:458] Webserver started at http://127.19.180.60:39991/ using document root <none> and password file <none>
15:02:50.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.900686 20346 fs_manager.cc:362] Metadata directory not provided
15:02:50.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.900771 20346 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:02:50.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.901026 20346 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:02:50.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.903432 20346 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/master-2/data/instance:
15:02:50.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "528e2c60e3bc4a5aa2ba21cac324d896"
15:02:50.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:02:50 on dist-test-slave-b45r"
15:02:50.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.904260 20346 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/master-2/wal/instance:
15:02:50.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "528e2c60e3bc4a5aa2ba21cac324d896"
15:02:50.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:02:50 on dist-test-slave-b45r"
15:02:50.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.908821 20346 fs_manager.cc:696] Time spent creating directory manager: real 0.004s user 0.005s sys 0.000s
15:02:50.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.911767 20363 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:02:50.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.912889 20346 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.002s sys 0.001s
15:02:50.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.913015 20346 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/master-2/data,/tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/master-2/wal
15:02:50.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "528e2c60e3bc4a5aa2ba21cac324d896"
15:02:50.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:02:50 on dist-test-slave-b45r"
15:02:50.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.913177 20346 fs_report.cc:389] FS layout report
15:02:50.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:02:50.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/master-2/wal
15:02:50.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/master-2/wal
15:02:50.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/master-2/data/data
15:02:50.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:02:50.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:02:50.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:02:50.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:02:50.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:02:50.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:02:50.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:02:50.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:02:50.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:02:50.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:02:50.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:02:50.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.924288 20346 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:02:50.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.925034 20346 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:02:50.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.925267 20346 kserver.cc:163] Server-wide thread pool size limit: 3276
15:02:50.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.941718 20344 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.180.60" port: 32797 } attempt: 4
15:02:50.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.943321 20346 rpc_server.cc:307] RPC server started. Bound to: 127.19.180.60:32797
15:02:50.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.943339 20424 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.180.60:32797 every 8 connection(s)
15:02:50.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.944715 20263 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.180.60" port: 32797 } attempt: 2
15:02:50.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.945135 20346 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/master-2/data/info.pb
15:02:50.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.949669 20176 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskDlDeJ5/build/asan/bin/kudu as pid 20346
15:02:50.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.949857 20176 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/master-2/wal/instance
15:02:50.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.967288 20344 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ff11b88709b14cda87f5cecd80a792d6: Bootstrap starting.
15:02:50.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.968158 20427 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:02:50.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.971141 20344 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P ff11b88709b14cda87f5cecd80a792d6: Neither blocks nor log segments found. Creating new log.
15:02:50.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.973841 20344 log.cc:826] T 00000000000000000000000000000000 P ff11b88709b14cda87f5cecd80a792d6: Log is configured to *not* fsync() on all Append() calls
15:02:50.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.975627 20263 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1: Bootstrap starting.
15:02:50.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.975826 20427 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.180.62" port: 46081 } has no permanent_uuid. Determining permanent_uuid...
15:02:50.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.981561 20263 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1: Neither blocks nor log segments found. Creating new log.
15:02:50.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.983027 20263 log.cc:826] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1: Log is configured to *not* fsync() on all Append() calls
15:02:50.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.989517 20344 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ff11b88709b14cda87f5cecd80a792d6: No bootstrap required, opened a new log
15:02:50.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.991273 20263 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1: No bootstrap required, opened a new log
15:02:50.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.994488 20427 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.180.61" port: 37725 } has no permanent_uuid. Determining permanent_uuid...
15:02:50.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.997289 20344 raft_consensus.cc:357] T 00000000000000000000000000000000 P ff11b88709b14cda87f5cecd80a792d6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3ea36a1a750c40b98e9e225442fadbd1" member_type: VOTER last_known_addr { host: "127.19.180.62" port: 46081 } } peers { permanent_uuid: "ff11b88709b14cda87f5cecd80a792d6" member_type: VOTER last_known_addr { host: "127.19.180.61" port: 37725 } } peers { permanent_uuid: "528e2c60e3bc4a5aa2ba21cac324d896" member_type: VOTER last_known_addr { host: "127.19.180.60" port: 32797 } }
15:02:50.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.997709 20344 raft_consensus.cc:383] T 00000000000000000000000000000000 P ff11b88709b14cda87f5cecd80a792d6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:02:50.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.997925 20344 raft_consensus.cc:738] T 00000000000000000000000000000000 P ff11b88709b14cda87f5cecd80a792d6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ff11b88709b14cda87f5cecd80a792d6, State: Initialized, Role: FOLLOWER
15:02:50.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.998788 20344 consensus_queue.cc:260] T 00000000000000000000000000000000 P ff11b88709b14cda87f5cecd80a792d6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3ea36a1a750c40b98e9e225442fadbd1" member_type: VOTER last_known_addr { host: "127.19.180.62" port: 46081 } } peers { permanent_uuid: "ff11b88709b14cda87f5cecd80a792d6" member_type: VOTER last_known_addr { host: "127.19.180.61" port: 37725 } } peers { permanent_uuid: "528e2c60e3bc4a5aa2ba21cac324d896" member_type: VOTER last_known_addr { host: "127.19.180.60" port: 32797 } }
15:02:50.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.999663 20433 sys_catalog.cc:455] T 00000000000000000000000000000000 P ff11b88709b14cda87f5cecd80a792d6 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3ea36a1a750c40b98e9e225442fadbd1" member_type: VOTER last_known_addr { host: "127.19.180.62" port: 46081 } } peers { permanent_uuid: "ff11b88709b14cda87f5cecd80a792d6" member_type: VOTER last_known_addr { host: "127.19.180.61" port: 37725 } } peers { permanent_uuid: "528e2c60e3bc4a5aa2ba21cac324d896" member_type: VOTER last_known_addr { host: "127.19.180.60" port: 32797 } } }
15:02:51.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.999940 20433 sys_catalog.cc:458] T 00000000000000000000000000000000 P ff11b88709b14cda87f5cecd80a792d6 [sys.catalog]: This master's current role is: FOLLOWER
15:02:51.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:50.999917 20263 raft_consensus.cc:357] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3ea36a1a750c40b98e9e225442fadbd1" member_type: VOTER last_known_addr { host: "127.19.180.62" port: 46081 } } peers { permanent_uuid: "ff11b88709b14cda87f5cecd80a792d6" member_type: VOTER last_known_addr { host: "127.19.180.61" port: 37725 } } peers { permanent_uuid: "528e2c60e3bc4a5aa2ba21cac324d896" member_type: VOTER last_known_addr { host: "127.19.180.60" port: 32797 } }
15:02:51.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.000231 20263 raft_consensus.cc:383] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:02:51.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.000599 20263 raft_consensus.cc:738] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3ea36a1a750c40b98e9e225442fadbd1, State: Initialized, Role: FOLLOWER
15:02:51.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.001255 20263 consensus_queue.cc:260] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3ea36a1a750c40b98e9e225442fadbd1" member_type: VOTER last_known_addr { host: "127.19.180.62" port: 46081 } } peers { permanent_uuid: "ff11b88709b14cda87f5cecd80a792d6" member_type: VOTER last_known_addr { host: "127.19.180.61" port: 37725 } } peers { permanent_uuid: "528e2c60e3bc4a5aa2ba21cac324d896" member_type: VOTER last_known_addr { host: "127.19.180.60" port: 32797 } }
15:02:51.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.001616 20344 sys_catalog.cc:564] T 00000000000000000000000000000000 P ff11b88709b14cda87f5cecd80a792d6 [sys.catalog]: configured and running, proceeding with master startup.
15:02:51.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.002280 20427 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.180.60" port: 32797 } has no permanent_uuid. Determining permanent_uuid...
15:02:51.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.002569 20434 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3ea36a1a750c40b98e9e225442fadbd1" member_type: VOTER last_known_addr { host: "127.19.180.62" port: 46081 } } peers { permanent_uuid: "ff11b88709b14cda87f5cecd80a792d6" member_type: VOTER last_known_addr { host: "127.19.180.61" port: 37725 } } peers { permanent_uuid: "528e2c60e3bc4a5aa2ba21cac324d896" member_type: VOTER last_known_addr { host: "127.19.180.60" port: 32797 } } }
15:02:51.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.002841 20434 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1 [sys.catalog]: This master's current role is: FOLLOWER
15:02:51.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.003835 20263 sys_catalog.cc:564] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1 [sys.catalog]: configured and running, proceeding with master startup.
15:02:51.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.013660 20427 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896: Bootstrap starting.
15:02:51.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.017993 20427 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896: Neither blocks nor log segments found. Creating new log.
15:02:51.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.019394 20427 log.cc:826] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896: Log is configured to *not* fsync() on all Append() calls
15:02:51.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:51.026546 20449 catalog_manager.cc:1559] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:02:51.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:51.026670 20449 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:02:51.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.028796 20427 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896: No bootstrap required, opened a new log
15:02:51.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.022035 20187 master_runner.cc:186] Error getting master registration for 127.19.180.61:37725: OK, instance_id { permanent_uuid: "ff11b88709b14cda87f5cecd80a792d6" instance_seqno: 1736694170545499 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:02:51.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.032305 20187 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:02:51.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:51.033381 20288 tablet.cc:2367] T 00000000000000000000000000000000 P ff11b88709b14cda87f5cecd80a792d6: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:02:51.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.037652 20433 raft_consensus.cc:491] T 00000000000000000000000000000000 P ff11b88709b14cda87f5cecd80a792d6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:02:51.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.037664 20427 raft_consensus.cc:357] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3ea36a1a750c40b98e9e225442fadbd1" member_type: VOTER last_known_addr { host: "127.19.180.62" port: 46081 } } peers { permanent_uuid: "ff11b88709b14cda87f5cecd80a792d6" member_type: VOTER last_known_addr { host: "127.19.180.61" port: 37725 } } peers { permanent_uuid: "528e2c60e3bc4a5aa2ba21cac324d896" member_type: VOTER last_known_addr { host: "127.19.180.60" port: 32797 } }
15:02:51.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.038049 20427 raft_consensus.cc:383] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:02:51.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.038179 20427 raft_consensus.cc:738] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 528e2c60e3bc4a5aa2ba21cac324d896, State: Initialized, Role: FOLLOWER
15:02:51.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.038470 20433 raft_consensus.cc:513] T 00000000000000000000000000000000 P ff11b88709b14cda87f5cecd80a792d6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3ea36a1a750c40b98e9e225442fadbd1" member_type: VOTER last_known_addr { host: "127.19.180.62" port: 46081 } } peers { permanent_uuid: "ff11b88709b14cda87f5cecd80a792d6" member_type: VOTER last_known_addr { host: "127.19.180.61" port: 37725 } } peers { permanent_uuid: "528e2c60e3bc4a5aa2ba21cac324d896" member_type: VOTER last_known_addr { host: "127.19.180.60" port: 32797 } }
15:02:51.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.038841 20427 consensus_queue.cc:260] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3ea36a1a750c40b98e9e225442fadbd1" member_type: VOTER last_known_addr { host: "127.19.180.62" port: 46081 } } peers { permanent_uuid: "ff11b88709b14cda87f5cecd80a792d6" member_type: VOTER last_known_addr { host: "127.19.180.61" port: 37725 } } peers { permanent_uuid: "528e2c60e3bc4a5aa2ba21cac324d896" member_type: VOTER last_known_addr { host: "127.19.180.60" port: 32797 } }
15:02:51.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.040984 20228 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ff11b88709b14cda87f5cecd80a792d6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3ea36a1a750c40b98e9e225442fadbd1" is_pre_election: true
15:02:51.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.041577 20228 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ff11b88709b14cda87f5cecd80a792d6 in term 0.
15:02:51.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.042202 20286 leader_election.cc:304] T 00000000000000000000000000000000 P ff11b88709b14cda87f5cecd80a792d6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3ea36a1a750c40b98e9e225442fadbd1, ff11b88709b14cda87f5cecd80a792d6; no voters:
15:02:51.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.046281 20456 sys_catalog.cc:455] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3ea36a1a750c40b98e9e225442fadbd1" member_type: VOTER last_known_addr { host: "127.19.180.62" port: 46081 } } peers { permanent_uuid: "ff11b88709b14cda87f5cecd80a792d6" member_type: VOTER last_known_addr { host: "127.19.180.61" port: 37725 } } peers { permanent_uuid: "528e2c60e3bc4a5aa2ba21cac324d896" member_type: VOTER last_known_addr { host: "127.19.180.60" port: 32797 } } }
15:02:51.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.046665 20456 sys_catalog.cc:458] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [sys.catalog]: This master's current role is: FOLLOWER
15:02:51.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.047618 20427 sys_catalog.cc:564] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [sys.catalog]: configured and running, proceeding with master startup.
15:02:51.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.049790 20433 leader_election.cc:290] T 00000000000000000000000000000000 P ff11b88709b14cda87f5cecd80a792d6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3ea36a1a750c40b98e9e225442fadbd1 (127.19.180.62:46081), 528e2c60e3bc4a5aa2ba21cac324d896 (127.19.180.60:32797)
15:02:51.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.050493 20433 raft_consensus.cc:2798] T 00000000000000000000000000000000 P ff11b88709b14cda87f5cecd80a792d6 [term 0 FOLLOWER]: Leader pre-election won for term 1
15:02:51.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.050990 20433 raft_consensus.cc:491] T 00000000000000000000000000000000 P ff11b88709b14cda87f5cecd80a792d6 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
15:02:51.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.051342 20433 raft_consensus.cc:3054] T 00000000000000000000000000000000 P ff11b88709b14cda87f5cecd80a792d6 [term 0 FOLLOWER]: Advancing to term 1
15:02:51.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.052748 20461 catalog_manager.cc:796] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
15:02:51.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.055476 20433 raft_consensus.cc:513] T 00000000000000000000000000000000 P ff11b88709b14cda87f5cecd80a792d6 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3ea36a1a750c40b98e9e225442fadbd1" member_type: VOTER last_known_addr { host: "127.19.180.62" port: 46081 } } peers { permanent_uuid: "ff11b88709b14cda87f5cecd80a792d6" member_type: VOTER last_known_addr { host: "127.19.180.61" port: 37725 } } peers { permanent_uuid: "528e2c60e3bc4a5aa2ba21cac324d896" member_type: VOTER last_known_addr { host: "127.19.180.60" port: 32797 } }
15:02:51.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.056123 20433 leader_election.cc:290] T 00000000000000000000000000000000 P ff11b88709b14cda87f5cecd80a792d6 [CANDIDATE]: Term 1 election: Requested vote from peers 3ea36a1a750c40b98e9e225442fadbd1 (127.19.180.62:46081), 528e2c60e3bc4a5aa2ba21cac324d896 (127.19.180.60:32797)
15:02:51.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.057228 20390 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ff11b88709b14cda87f5cecd80a792d6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "528e2c60e3bc4a5aa2ba21cac324d896" is_pre_election: true
15:02:51.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.057739 20390 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ff11b88709b14cda87f5cecd80a792d6 in term 0.
15:02:51.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.058513 20228 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ff11b88709b14cda87f5cecd80a792d6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3ea36a1a750c40b98e9e225442fadbd1"
15:02:51.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.058743 20228 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1 [term 0 FOLLOWER]: Advancing to term 1
15:02:51.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.061053 20390 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ff11b88709b14cda87f5cecd80a792d6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "528e2c60e3bc4a5aa2ba21cac324d896"
15:02:51.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.061242 20390 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [term 0 FOLLOWER]: Advancing to term 1
15:02:51.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.061791 20228 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ff11b88709b14cda87f5cecd80a792d6 in term 1.
15:02:51.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.063608 20390 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ff11b88709b14cda87f5cecd80a792d6 in term 1.
15:02:51.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.064743 20286 leader_election.cc:304] T 00000000000000000000000000000000 P ff11b88709b14cda87f5cecd80a792d6 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 528e2c60e3bc4a5aa2ba21cac324d896, ff11b88709b14cda87f5cecd80a792d6; no voters:
15:02:51.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.065135 20458 raft_consensus.cc:2798] T 00000000000000000000000000000000 P ff11b88709b14cda87f5cecd80a792d6 [term 1 FOLLOWER]: Leader election won for term 1
15:02:51.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.065325 20458 raft_consensus.cc:695] T 00000000000000000000000000000000 P ff11b88709b14cda87f5cecd80a792d6 [term 1 LEADER]: Becoming Leader. State: Replica: ff11b88709b14cda87f5cecd80a792d6, State: Running, Role: LEADER
15:02:51.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.065850 20458 consensus_queue.cc:237] T 00000000000000000000000000000000 P ff11b88709b14cda87f5cecd80a792d6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3ea36a1a750c40b98e9e225442fadbd1" member_type: VOTER last_known_addr { host: "127.19.180.62" port: 46081 } } peers { permanent_uuid: "ff11b88709b14cda87f5cecd80a792d6" member_type: VOTER last_known_addr { host: "127.19.180.61" port: 37725 } } peers { permanent_uuid: "528e2c60e3bc4a5aa2ba21cac324d896" member_type: VOTER last_known_addr { host: "127.19.180.60" port: 32797 } }
15:02:51.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.073957 20433 sys_catalog.cc:455] T 00000000000000000000000000000000 P ff11b88709b14cda87f5cecd80a792d6 [sys.catalog]: SysCatalogTable state changed. Reason: New leader ff11b88709b14cda87f5cecd80a792d6. Latest consensus state: current_term: 1 leader_uuid: "ff11b88709b14cda87f5cecd80a792d6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3ea36a1a750c40b98e9e225442fadbd1" member_type: VOTER last_known_addr { host: "127.19.180.62" port: 46081 } } peers { permanent_uuid: "ff11b88709b14cda87f5cecd80a792d6" member_type: VOTER last_known_addr { host: "127.19.180.61" port: 37725 } } peers { permanent_uuid: "528e2c60e3bc4a5aa2ba21cac324d896" member_type: VOTER last_known_addr { host: "127.19.180.60" port: 32797 } } }
15:02:51.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.074365 20433 sys_catalog.cc:458] T 00000000000000000000000000000000 P ff11b88709b14cda87f5cecd80a792d6 [sys.catalog]: This master's current role is: LEADER
15:02:51.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.076006 20265 master_runner.cc:186] Error getting master registration for 127.19.180.60:32797: OK, instance_id { permanent_uuid: "528e2c60e3bc4a5aa2ba21cac324d896" instance_seqno: 1736694170938641 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:02:51.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.083341 20265 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:02:51.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.086062 20469 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
15:02:51.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:51.097033 20473 catalog_manager.cc:1559] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:02:51.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:51.097180 20473 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:02:51.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.102294 20469 catalog_manager.cc:1485] Initializing Kudu cluster ID...
15:02:51.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:51.110107 20207 tablet.cc:2367] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:02:51.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.124645 20390 raft_consensus.cc:1270] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [term 1 FOLLOWER]: Refusing update from remote peer ff11b88709b14cda87f5cecd80a792d6: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
15:02:51.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.125742 20433 consensus_queue.cc:1035] T 00000000000000000000000000000000 P ff11b88709b14cda87f5cecd80a792d6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "528e2c60e3bc4a5aa2ba21cac324d896" member_type: VOTER last_known_addr { host: "127.19.180.60" port: 32797 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:02:51.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.149158 20228 raft_consensus.cc:1270] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1 [term 1 FOLLOWER]: Refusing update from remote peer ff11b88709b14cda87f5cecd80a792d6: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
15:02:51.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.150221 20433 consensus_queue.cc:1035] T 00000000000000000000000000000000 P ff11b88709b14cda87f5cecd80a792d6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3ea36a1a750c40b98e9e225442fadbd1" member_type: VOTER last_known_addr { host: "127.19.180.62" port: 46081 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:02:51.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.167680 20456 sys_catalog.cc:455] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [sys.catalog]: SysCatalogTable state changed. Reason: New leader ff11b88709b14cda87f5cecd80a792d6. Latest consensus state: current_term: 1 leader_uuid: "ff11b88709b14cda87f5cecd80a792d6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3ea36a1a750c40b98e9e225442fadbd1" member_type: VOTER last_known_addr { host: "127.19.180.62" port: 46081 } } peers { permanent_uuid: "ff11b88709b14cda87f5cecd80a792d6" member_type: VOTER last_known_addr { host: "127.19.180.61" port: 37725 } } peers { permanent_uuid: "528e2c60e3bc4a5aa2ba21cac324d896" member_type: VOTER last_known_addr { host: "127.19.180.60" port: 32797 } } }
15:02:51.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.167973 20456 sys_catalog.cc:458] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [sys.catalog]: This master's current role is: FOLLOWER
15:02:51.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.170524 20476 mvcc.cc:204] Tried to move back new op lower bound from 7113499324915208192 to 7113499324690710528. Current Snapshot: MvccSnapshot[applied={T|T < 7113499324915208192}]
15:02:51.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.172035 20477 mvcc.cc:204] Tried to move back new op lower bound from 7113499324915208192 to 7113499324690710528. Current Snapshot: MvccSnapshot[applied={T|T < 7113499324915208192}]
15:02:51.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.190632 20456 sys_catalog.cc:455] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "ff11b88709b14cda87f5cecd80a792d6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3ea36a1a750c40b98e9e225442fadbd1" member_type: VOTER last_known_addr { host: "127.19.180.62" port: 46081 } } peers { permanent_uuid: "ff11b88709b14cda87f5cecd80a792d6" member_type: VOTER last_known_addr { host: "127.19.180.61" port: 37725 } } peers { permanent_uuid: "528e2c60e3bc4a5aa2ba21cac324d896" member_type: VOTER last_known_addr { host: "127.19.180.60" port: 32797 } } }
15:02:51.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.190847 20456 sys_catalog.cc:458] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [sys.catalog]: This master's current role is: FOLLOWER
15:02:51.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.194132 20458 sys_catalog.cc:455] T 00000000000000000000000000000000 P ff11b88709b14cda87f5cecd80a792d6 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "ff11b88709b14cda87f5cecd80a792d6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3ea36a1a750c40b98e9e225442fadbd1" member_type: VOTER last_known_addr { host: "127.19.180.62" port: 46081 } } peers { permanent_uuid: "ff11b88709b14cda87f5cecd80a792d6" member_type: VOTER last_known_addr { host: "127.19.180.61" port: 37725 } } peers { permanent_uuid: "528e2c60e3bc4a5aa2ba21cac324d896" member_type: VOTER last_known_addr { host: "127.19.180.60" port: 32797 } } }
15:02:51.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.194455 20458 sys_catalog.cc:458] T 00000000000000000000000000000000 P ff11b88709b14cda87f5cecd80a792d6 [sys.catalog]: This master's current role is: LEADER
15:02:51.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.196655 20469 catalog_manager.cc:1348] Generated new cluster ID: e212c04cc2fe4a68b45af340d61c59d7
15:02:51.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.196760 20469 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
15:02:51.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.199986 20433 sys_catalog.cc:455] T 00000000000000000000000000000000 P ff11b88709b14cda87f5cecd80a792d6 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "ff11b88709b14cda87f5cecd80a792d6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3ea36a1a750c40b98e9e225442fadbd1" member_type: VOTER last_known_addr { host: "127.19.180.62" port: 46081 } } peers { permanent_uuid: "ff11b88709b14cda87f5cecd80a792d6" member_type: VOTER last_known_addr { host: "127.19.180.61" port: 37725 } } peers { permanent_uuid: "528e2c60e3bc4a5aa2ba21cac324d896" member_type: VOTER last_known_addr { host: "127.19.180.60" port: 32797 } } }
15:02:51.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.200224 20433 sys_catalog.cc:458] T 00000000000000000000000000000000 P ff11b88709b14cda87f5cecd80a792d6 [sys.catalog]: This master's current role is: LEADER
15:02:51.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.200068 20434 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1 [sys.catalog]: SysCatalogTable state changed. Reason: New leader ff11b88709b14cda87f5cecd80a792d6. Latest consensus state: current_term: 1 leader_uuid: "ff11b88709b14cda87f5cecd80a792d6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3ea36a1a750c40b98e9e225442fadbd1" member_type: VOTER last_known_addr { host: "127.19.180.62" port: 46081 } } peers { permanent_uuid: "ff11b88709b14cda87f5cecd80a792d6" member_type: VOTER last_known_addr { host: "127.19.180.61" port: 37725 } } peers { permanent_uuid: "528e2c60e3bc4a5aa2ba21cac324d896" member_type: VOTER last_known_addr { host: "127.19.180.60" port: 32797 } } }
15:02:51.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.200361 20434 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1 [sys.catalog]: This master's current role is: FOLLOWER
15:02:51.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.208099 20434 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "ff11b88709b14cda87f5cecd80a792d6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3ea36a1a750c40b98e9e225442fadbd1" member_type: VOTER last_known_addr { host: "127.19.180.62" port: 46081 } } peers { permanent_uuid: "ff11b88709b14cda87f5cecd80a792d6" member_type: VOTER last_known_addr { host: "127.19.180.61" port: 37725 } } peers { permanent_uuid: "528e2c60e3bc4a5aa2ba21cac324d896" member_type: VOTER last_known_addr { host: "127.19.180.60" port: 32797 } } }
15:02:51.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.208326 20434 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1 [sys.catalog]: This master's current role is: FOLLOWER
15:02:51.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.240320 20469 catalog_manager.cc:1371] Generated new certificate authority record
15:02:51.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.242064 20469 catalog_manager.cc:1505] Loading token signing keys...
15:02:51.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.262202 20469 catalog_manager.cc:5899] T 00000000000000000000000000000000 P ff11b88709b14cda87f5cecd80a792d6: Generated new TSK 0
15:02:51.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.263249 20469 catalog_manager.cc:1515] Initializing in-progress tserver states...
15:02:51.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.282581 20176 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskDlDeJ5/build/asan/bin/kudu
15:02:51.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDlDeJ5/build/asan/bin/kudu
15:02:51.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/ts-0/wal
15:02:51.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/ts-0/data
15:02:51.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:02:51.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:02:51.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:02:51.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:02:51.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:02:51.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:02:51.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/ts-0/logs
15:02:51.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/ts-0/data/info.pb
15:02:51.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:02:51.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:02:51.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:02:51.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:02:51.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:02:51.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:02:51.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:02:51.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:02:51.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:02:51.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:02:51.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.180.1:0
15:02:51.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.180.1
15:02:51.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.180.1
15:02:51.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:02:51.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797
15:02:51.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.180.20:41217
15:02:51.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:02:51.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:02:51.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:02:51.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
15:02:51.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
15:02:51.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
15:02:51.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:51.738895 20484 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:02:51.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:51.739228 20484 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:02:51.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:51.739373 20484 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:02:51.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:51.746903 20484 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:02:51.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:51.747087 20484 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
15:02:51.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:51.747140 20484 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
15:02:51.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:51.747200 20484 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
15:02:51.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:51.747325 20484 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.19.180.1
15:02:51.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.756372 20484 tablet_server_runner.cc:78] Tablet server non-default flags:
15:02:51.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:02:51.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.180.20:41217
15:02:51.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:02:51.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:02:51.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/ts-0/data
15:02:51.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/ts-0/wal
15:02:51.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:02:51.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:02:51.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.180.1:0
15:02:51.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:02:51.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:02:51.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:02:51.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/ts-0/data/info.pb
15:02:51.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.180.1
15:02:51.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:02:51.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
15:02:51.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
15:02:51.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
15:02:51.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797
15:02:51.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:02:51.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:02:51.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:02:51.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:02:51.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:02:51.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.180.1
15:02:51.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/ts-0/logs
15:02:51.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:02:51.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:02:51.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:02:51.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:02:51.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:02:51.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:02:51.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:02:51.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Jan 2025 14:43:22 UTC on 24a791456cd2
15:02:51.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4143
15:02:51.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
15:02:51.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.758193 20484 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:02:51.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.759956 20484 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:02:51.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:51.771085 20490 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:02:51.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:51.771553 20489 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:02:51.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:51.771947 20492 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:02:51.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.772203 20484 server_base.cc:1034] running on GCE node
15:02:51.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.772737 20484 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:02:51.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.773586 20484 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:02:51.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.774802 20484 hybrid_clock.cc:648] HybridClock initialized: now 1736694171774751 us; error 40 us; skew 500 ppm
15:02:51.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.775120 20484 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:02:51.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.781533 20484 webserver.cc:458] Webserver started at http://127.19.180.1:39975/ using document root <none> and password file <none>
15:02:51.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.782260 20484 fs_manager.cc:362] Metadata directory not provided
15:02:51.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.782364 20484 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:02:51.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.782672 20484 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:02:51.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.785162 20484 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/ts-0/data/instance:
15:02:51.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a0258fb56e9d4b858dfa55326a152559"
15:02:51.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:02:51 on dist-test-slave-b45r"
15:02:51.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.785919 20484 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/ts-0/wal/instance:
15:02:51.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a0258fb56e9d4b858dfa55326a152559"
15:02:51.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:02:51 on dist-test-slave-b45r"
15:02:51.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.790702 20484 fs_manager.cc:696] Time spent creating directory manager: real 0.004s user 0.001s sys 0.006s
15:02:51.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.793720 20498 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:02:51.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.794894 20484 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.000s sys 0.002s
15:02:51.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.795053 20484 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/ts-0/data,/tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/ts-0/wal
15:02:51.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a0258fb56e9d4b858dfa55326a152559"
15:02:51.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:02:51 on dist-test-slave-b45r"
15:02:51.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.795214 20484 fs_report.cc:389] FS layout report
15:02:51.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:02:51.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/ts-0/wal
15:02:51.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/ts-0/wal
15:02:51.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/ts-0/data/data
15:02:51.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:02:51.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:02:51.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:02:51.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:02:51.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:02:51.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:02:51.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:02:51.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:02:51.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:02:51.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:02:51.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:02:51.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.806542 20484 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:02:51.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.807258 20484 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:02:51.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.807485 20484 kserver.cc:163] Server-wide thread pool size limit: 3276
15:02:51.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.810007 20484 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:02:51.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.810125 20484 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
15:02:51.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.810230 20484 ts_tablet_manager.cc:610] Registered 0 tablets
15:02:51.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.810299 20484 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:02:51.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.860883 20484 rpc_server.cc:307] RPC server started. Bound to: 127.19.180.1:46735
15:02:51.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.860913 20621 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.180.1:46735 every 8 connection(s)
15:02:51.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.862294 20484 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/ts-0/data/info.pb
15:02:51.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.864342 20623 heartbeater.cc:346] Connected to a master server at 127.19.180.62:46081
15:02:51.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.864740 20623 heartbeater.cc:463] Registering TS with master...
15:02:51.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.865854 20623 heartbeater.cc:510] Master 127.19.180.62:46081 requested a full tablet report, sending...
15:02:51.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.868139 20218 ts_manager.cc:194] Registered new tserver with Master: a0258fb56e9d4b858dfa55326a152559 (127.19.180.1:46735)
15:02:51.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.870962 20176 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskDlDeJ5/build/asan/bin/kudu as pid 20484
15:02:51.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.871198 20176 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/ts-0/wal/instance
15:02:51.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.875743 20176 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskDlDeJ5/build/asan/bin/kudu
15:02:51.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDlDeJ5/build/asan/bin/kudu
15:02:51.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/ts-1/wal
15:02:51.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/ts-1/data
15:02:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:02:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:02:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:02:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:02:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:02:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:02:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/ts-1/logs
15:02:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/ts-1/data/info.pb
15:02:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:02:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:02:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:02:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:02:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:02:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:02:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:02:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:02:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:02:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:02:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.180.2:0
15:02:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.180.2
15:02:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.180.2
15:02:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:02:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797
15:02:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.180.20:41217
15:02:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:02:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:02:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:02:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
15:02:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
15:02:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
15:02:51.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.879482 20622 heartbeater.cc:346] Connected to a master server at 127.19.180.60:32797
15:02:51.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.879860 20622 heartbeater.cc:463] Registering TS with master...
15:02:51.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.880285 20622 heartbeater.cc:510] Master 127.19.180.60:32797 requested a full tablet report, sending...
15:02:51.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.881804 20380 ts_manager.cc:194] Registered new tserver with Master: a0258fb56e9d4b858dfa55326a152559 (127.19.180.1:46735)
15:02:51.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.882473 20624 heartbeater.cc:346] Connected to a master server at 127.19.180.61:37725
15:02:51.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.882613 20624 heartbeater.cc:463] Registering TS with master...
15:02:51.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.883064 20624 heartbeater.cc:510] Master 127.19.180.61:37725 requested a full tablet report, sending...
15:02:51.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.884464 20299 ts_manager.cc:194] Registered new tserver with Master: a0258fb56e9d4b858dfa55326a152559 (127.19.180.1:46735)
15:02:51.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:51.886406 20299 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.19.180.1:45085
15:02:52.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.029245 20449 catalog_manager.cc:1260] Loaded cluster ID: e212c04cc2fe4a68b45af340d61c59d7
15:02:52.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.029376 20449 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1: loading cluster ID for follower catalog manager: success
15:02:52.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.033160 20449 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1: acquiring CA information for follower catalog manager: success
15:02:52.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.035492 20449 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:02:52.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.099053 20473 catalog_manager.cc:1260] Loaded cluster ID: e212c04cc2fe4a68b45af340d61c59d7
15:02:52.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.099148 20473 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896: loading cluster ID for follower catalog manager: success
15:02:52.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.101828 20473 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896: acquiring CA information for follower catalog manager: success
15:02:52.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.103734 20473 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:02:52.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:52.192764 20627 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:02:52.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:52.193023 20627 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:02:52.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:52.193126 20627 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:02:52.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:52.200964 20627 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:02:52.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:52.201102 20627 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
15:02:52.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:52.201138 20627 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
15:02:52.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:52.201182 20627 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
15:02:52.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:52.201274 20627 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.19.180.2
15:02:52.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.210412 20627 tablet_server_runner.cc:78] Tablet server non-default flags:
15:02:52.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:02:52.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.180.20:41217
15:02:52.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:02:52.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:02:52.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/ts-1/data
15:02:52.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/ts-1/wal
15:02:52.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:02:52.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:02:52.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.180.2:0
15:02:52.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:02:52.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:02:52.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:02:52.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/ts-1/data/info.pb
15:02:52.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.180.2
15:02:52.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:02:52.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
15:02:52.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
15:02:52.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
15:02:52.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797
15:02:52.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:02:52.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:02:52.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:02:52.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:02:52.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:02:52.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.180.2
15:02:52.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/ts-1/logs
15:02:52.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:02:52.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:02:52.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:02:52.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:02:52.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:02:52.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:02:52.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:02:52.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Jan 2025 14:43:22 UTC on 24a791456cd2
15:02:52.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4143
15:02:52.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
15:02:52.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.211922 20627 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:02:52.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.213696 20627 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:02:52.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:52.224824 20632 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:02:52.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:52.225340 20633 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:02:52.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:52.225826 20635 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:02:52.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.226073 20627 server_base.cc:1034] running on GCE node
15:02:52.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.226652 20627 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:02:52.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.227582 20627 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:02:52.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.228785 20627 hybrid_clock.cc:648] HybridClock initialized: now 1736694172228731 us; error 45 us; skew 500 ppm
15:02:52.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.229082 20627 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:02:52.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.231189 20627 webserver.cc:458] Webserver started at http://127.19.180.2:40739/ using document root <none> and password file <none>
15:02:52.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.231899 20627 fs_manager.cc:362] Metadata directory not provided
15:02:52.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.232002 20627 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:02:52.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.232297 20627 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:02:52.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.234733 20627 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/ts-1/data/instance:
15:02:52.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1ab780ab6f274ded9a5fdac3c28998c7"
15:02:52.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:02:52 on dist-test-slave-b45r"
15:02:52.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.235543 20627 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/ts-1/wal/instance:
15:02:52.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1ab780ab6f274ded9a5fdac3c28998c7"
15:02:52.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:02:52 on dist-test-slave-b45r"
15:02:52.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.240176 20627 fs_manager.cc:696] Time spent creating directory manager: real 0.004s user 0.000s sys 0.004s
15:02:52.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.243155 20641 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:02:52.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.244340 20627 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.002s sys 0.001s
15:02:52.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.244488 20627 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/ts-1/data,/tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/ts-1/wal
15:02:52.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1ab780ab6f274ded9a5fdac3c28998c7"
15:02:52.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:02:52 on dist-test-slave-b45r"
15:02:52.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.244674 20627 fs_report.cc:389] FS layout report
15:02:52.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:02:52.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/ts-1/wal
15:02:52.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/ts-1/wal
15:02:52.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/ts-1/data/data
15:02:52.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:02:52.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:02:52.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:02:52.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:02:52.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:02:52.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:02:52.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:02:52.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:02:52.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:02:52.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:02:52.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:02:52.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.264531 20627 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:02:52.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.265266 20627 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:02:52.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.265532 20627 kserver.cc:163] Server-wide thread pool size limit: 3276
15:02:52.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.268175 20627 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:02:52.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.268268 20627 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
15:02:52.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.268393 20627 ts_tablet_manager.cc:610] Registered 0 tablets
15:02:52.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.268465 20627 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:02:52.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.326222 20627 rpc_server.cc:307] RPC server started. Bound to: 127.19.180.2:35451
15:02:52.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.326262 20762 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.180.2:35451 every 8 connection(s)
15:02:52.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.327503 20627 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/ts-1/data/info.pb
15:02:52.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.330590 20764 heartbeater.cc:346] Connected to a master server at 127.19.180.62:46081
15:02:52.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.330869 20764 heartbeater.cc:463] Registering TS with master...
15:02:52.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.331925 20764 heartbeater.cc:510] Master 127.19.180.62:46081 requested a full tablet report, sending...
15:02:52.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.333451 20218 ts_manager.cc:194] Registered new tserver with Master: 1ab780ab6f274ded9a5fdac3c28998c7 (127.19.180.2:35451)
15:02:52.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.334977 20176 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskDlDeJ5/build/asan/bin/kudu as pid 20627
15:02:52.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.335140 20176 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/ts-1/wal/instance
15:02:52.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.338014 20765 heartbeater.cc:346] Connected to a master server at 127.19.180.61:37725
15:02:52.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.338145 20765 heartbeater.cc:463] Registering TS with master...
15:02:52.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.338532 20176 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskDlDeJ5/build/asan/bin/kudu
15:02:52.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDlDeJ5/build/asan/bin/kudu
15:02:52.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/ts-2/wal
15:02:52.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/ts-2/data
15:02:52.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:02:52.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:02:52.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:02:52.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:02:52.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:02:52.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:02:52.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/ts-2/logs
15:02:52.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/ts-2/data/info.pb
15:02:52.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:02:52.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:02:52.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:02:52.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:02:52.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:02:52.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:02:52.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:02:52.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:02:52.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:02:52.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:02:52.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.180.3:0
15:02:52.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.180.3
15:02:52.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.180.3
15:02:52.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:02:52.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797
15:02:52.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.180.20:41217
15:02:52.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:02:52.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:02:52.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:02:52.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
15:02:52.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
15:02:52.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
15:02:52.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.338657 20765 heartbeater.cc:510] Master 127.19.180.61:37725 requested a full tablet report, sending...
15:02:52.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.339028 20763 heartbeater.cc:346] Connected to a master server at 127.19.180.60:32797
15:02:52.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.339136 20763 heartbeater.cc:463] Registering TS with master...
15:02:52.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.339618 20763 heartbeater.cc:510] Master 127.19.180.60:32797 requested a full tablet report, sending...
15:02:52.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.339597 20299 ts_manager.cc:194] Registered new tserver with Master: 1ab780ab6f274ded9a5fdac3c28998c7 (127.19.180.2:35451)
15:02:52.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.340457 20380 ts_manager.cc:194] Registered new tserver with Master: 1ab780ab6f274ded9a5fdac3c28998c7 (127.19.180.2:35451)
15:02:52.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.340520 20299 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.19.180.2:36367
15:02:52.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:52.643654 20768 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:02:52.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:52.643906 20768 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:02:52.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:52.644007 20768 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:02:52.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:52.651448 20768 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:02:52.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:52.651582 20768 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
15:02:52.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:52.651625 20768 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
15:02:52.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:52.651669 20768 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
15:02:52.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:52.651755 20768 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.19.180.3
15:02:52.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.660434 20768 tablet_server_runner.cc:78] Tablet server non-default flags:
15:02:52.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:02:52.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.180.20:41217
15:02:52.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:02:52.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:02:52.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/ts-2/data
15:02:52.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/ts-2/wal
15:02:52.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:02:52.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:02:52.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.180.3:0
15:02:52.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:02:52.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:02:52.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:02:52.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/ts-2/data/info.pb
15:02:52.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.180.3
15:02:52.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:02:52.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
15:02:52.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
15:02:52.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
15:02:52.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797
15:02:52.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:02:52.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:02:52.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:02:52.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:02:52.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:02:52.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.180.3
15:02:52.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/ts-2/logs
15:02:52.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:02:52.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:02:52.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:02:52.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:02:52.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:02:52.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:02:52.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:02:52.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Jan 2025 14:43:22 UTC on 24a791456cd2
15:02:52.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4143
15:02:52.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
15:02:52.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.661837 20768 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:02:52.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.663489 20768 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:02:52.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:52.674604 20773 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:02:52.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:52.675091 20774 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:02:52.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:52.675441 20776 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:02:52.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.675765 20768 server_base.cc:1034] running on GCE node
15:02:52.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.676365 20768 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:02:52.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.677438 20768 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:02:52.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.678655 20768 hybrid_clock.cc:648] HybridClock initialized: now 1736694172678602 us; error 56 us; skew 500 ppm
15:02:52.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.678989 20768 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:02:52.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.681166 20768 webserver.cc:458] Webserver started at http://127.19.180.3:34345/ using document root <none> and password file <none>
15:02:52.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.681892 20768 fs_manager.cc:362] Metadata directory not provided
15:02:52.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.681993 20768 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:02:52.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.682313 20768 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:02:52.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.684774 20768 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/ts-2/data/instance:
15:02:52.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c45c25ab0a664200896372dcc7565f59"
15:02:52.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:02:52 on dist-test-slave-b45r"
15:02:52.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.685576 20768 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/ts-2/wal/instance:
15:02:52.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c45c25ab0a664200896372dcc7565f59"
15:02:52.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:02:52 on dist-test-slave-b45r"
15:02:52.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.690366 20768 fs_manager.cc:696] Time spent creating directory manager: real 0.004s user 0.005s sys 0.001s
15:02:52.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.693662 20782 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:02:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.694808 20768 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
15:02:52.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.694970 20768 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/ts-2/data,/tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/ts-2/wal
15:02:52.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c45c25ab0a664200896372dcc7565f59"
15:02:52.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:02:52 on dist-test-slave-b45r"
15:02:52.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.695158 20768 fs_report.cc:389] FS layout report
15:02:52.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:02:52.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/ts-2/wal
15:02:52.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/ts-2/wal
15:02:52.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/ts-2/data/data
15:02:52.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:02:52.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:02:52.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:02:52.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:02:52.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:02:52.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:02:52.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:02:52.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:02:52.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:02:52.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:02:52.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:02:52.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.725994 20768 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:02:52.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.726774 20768 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:02:52.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.727025 20768 kserver.cc:163] Server-wide thread pool size limit: 3276
15:02:52.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.729666 20768 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:02:52.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.729770 20768 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
15:02:52.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.729895 20768 ts_tablet_manager.cc:610] Registered 0 tablets
15:02:52.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.729964 20768 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:02:52.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.783231 20768 rpc_server.cc:307] RPC server started. Bound to: 127.19.180.3:44881
15:02:52.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.783311 20903 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.180.3:44881 every 8 connection(s)
15:02:52.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.784428 20768 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/ts-2/data/info.pb
15:02:52.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.788193 20905 heartbeater.cc:346] Connected to a master server at 127.19.180.62:46081
15:02:52.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.788473 20905 heartbeater.cc:463] Registering TS with master...
15:02:52.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.789372 20905 heartbeater.cc:510] Master 127.19.180.62:46081 requested a full tablet report, sending...
15:02:52.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.791890 20218 ts_manager.cc:194] Registered new tserver with Master: c45c25ab0a664200896372dcc7565f59 (127.19.180.3:44881)
15:02:52.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.792750 20176 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskDlDeJ5/build/asan/bin/kudu as pid 20768
15:02:52.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.792923 20176 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskDlDeJ5/test-tmp/mini-kudu-cluster1302668630860759119/ts-2/wal/instance
15:02:52.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.794728 20906 heartbeater.cc:346] Connected to a master server at 127.19.180.61:37725
15:02:52.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.794873 20906 heartbeater.cc:463] Registering TS with master...
15:02:52.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.794931 20904 heartbeater.cc:346] Connected to a master server at 127.19.180.60:32797
15:02:52.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.795079 20904 heartbeater.cc:463] Registering TS with master...
15:02:52.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.795506 20906 heartbeater.cc:510] Master 127.19.180.61:37725 requested a full tablet report, sending...
15:02:52.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.795583 20904 heartbeater.cc:510] Master 127.19.180.60:32797 requested a full tablet report, sending...
15:02:52.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.796415 20299 ts_manager.cc:194] Registered new tserver with Master: c45c25ab0a664200896372dcc7565f59 (127.19.180.3:44881)
15:02:52.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.796604 20379 ts_manager.cc:194] Registered new tserver with Master: c45c25ab0a664200896372dcc7565f59 (127.19.180.3:44881)
15:02:52.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.797268 20299 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.19.180.3:57973
15:02:52.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.800779 20176 external_mini_cluster.cc:934] 3 TS(s) registered with all masters
15:02:52.802 [DEBUG - main] (MiniKuduCluster.java:195) Response:
15:02:52.802 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
15:02:52.803 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.19.180.62"
port: 46081
}
bound_http_address {
host: "127.19.180.62"
port: 37911
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.19.180.61"
port: 37725
}
bound_http_address {
host: "127.19.180.61"
port: 41737
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.19.180.60"
port: 32797
}
bound_http_address {
host: "127.19.180.60"
port: 39991
}
}
}
15:02:52.803 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
15:02:52.804 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.19.180.1"
port: 46735
}
bound_http_address {
host: "127.19.180.1"
port: 39975
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.19.180.2"
port: 35451
}
bound_http_address {
host: "127.19.180.2"
port: 40739
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.19.180.3"
port: 44881
}
bound_http_address {
host: "127.19.180.3"
port: 34345
}
}
}
15:02:52.804 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
15:02:52.805 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
15:02:52.807 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.19.180.62:46081(127.19.180.62:46081)] connecting to peer
15:02:52.808 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.19.180.61:37725(127.19.180.61:37725)] connecting to peer
15:02:52.809 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.19.180.62:46081(127.19.180.62:46081)] Successfully connected to peer
15:02:52.809 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.19.180.60:32797(127.19.180.60:32797)] connecting to peer
15:02:52.810 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.19.180.61:37725(127.19.180.61:37725)] Successfully connected to peer
15:02:52.812 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.19.180.61
15:02:52.812 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.19.180.62
15:02:52.815 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.19.180.60:32797(127.19.180.60:32797)] Successfully connected to peer
15:02:52.816 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.19.180.60
15:02:52.823 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xe3d9ac17, L:/127.0.0.1:37786 - R:/127.19.180.61:37725] using SASL/PLAIN
15:02:52.825 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
15:02:52.826 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=19), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [19ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: OK, deferred=null) after lookup
15:02:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.827953 20299 catalog_manager.cc:1909] Servicing CreateTable request from {username='slave'} at 127.0.0.1:37786:
15:02:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_manager_fallback"
15:02:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
15:02:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:02:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
15:02:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
15:02:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
15:02:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
15:02:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
15:02:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
15:02:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
15:02:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:02:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:02:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column1_i"
15:02:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
15:02:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
15:02:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
15:02:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
15:02:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
15:02:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
15:02:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:02:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:02:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column2_i"
15:02:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
15:02:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
15:02:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
15:02:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
15:02:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
15:02:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
15:02:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:02:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:02:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column3_s"
15:02:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
15:02:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
15:02:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
15:02:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: DICT_ENCODING
15:02:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: LZ4
15:02:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 4096
15:02:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
15:02:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
15:02:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:02:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:02:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column4_b"
15:02:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: BOOL
15:02:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
15:02:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
15:02:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
15:02:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
15:02:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
15:02:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:02:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:02:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
15:02:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) hash_schema {
15:02:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:02:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
15:02:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:02:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_buckets: 2
15:02:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) seed: 0
15:02:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:02:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:02:52.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:52.830370 20299 catalog_manager.cc:6885] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table txn_manager_fallback in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
15:02:52.830 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xdee45138, L:/127.0.0.1:40764 - R:/127.19.180.60:32797] using SASL/PLAIN
15:02:52.832 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x7336ccbc, L:/127.0.0.1:51392 - R:/127.19.180.62:46081] using SASL/PLAIN
15:02:52.848 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table 221d15de79984a85af5c16c6e92777a4
15:02:52.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.888716 20624 heartbeater.cc:502] Master 127.19.180.61:37725 was elected leader, sending a full tablet report...
15:02:52.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.913259 20834 tablet_service.cc:1467] Processing CreateTablet for tablet 9727c0ccc99e478eb0c34096f44de1a6 (DEFAULT_TABLE table=txn_manager_fallback [id=221d15de79984a85af5c16c6e92777a4]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
15:02:52.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.913247 20695 tablet_service.cc:1467] Processing CreateTablet for tablet fb15a93555ef4c018b3028f224e8e83b (DEFAULT_TABLE table=txn_manager_fallback [id=221d15de79984a85af5c16c6e92777a4]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
15:02:52.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.914676 20834 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 9727c0ccc99e478eb0c34096f44de1a6. 1 dirs total, 0 dirs full, 0 dirs failed
15:02:52.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.914680 20695 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet fb15a93555ef4c018b3028f224e8e83b. 1 dirs total, 0 dirs full, 0 dirs failed
15:02:52.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.915012 20836 tablet_service.cc:1467] Processing CreateTablet for tablet fb15a93555ef4c018b3028f224e8e83b (DEFAULT_TABLE table=txn_manager_fallback [id=221d15de79984a85af5c16c6e92777a4]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
15:02:52.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.916452 20551 tablet_service.cc:1467] Processing CreateTablet for tablet fb15a93555ef4c018b3028f224e8e83b (DEFAULT_TABLE table=txn_manager_fallback [id=221d15de79984a85af5c16c6e92777a4]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
15:02:52.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.917568 20551 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet fb15a93555ef4c018b3028f224e8e83b. 1 dirs total, 0 dirs full, 0 dirs failed
15:02:52.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.917865 20836 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet fb15a93555ef4c018b3028f224e8e83b. 1 dirs total, 0 dirs full, 0 dirs failed
15:02:52.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.922073 20694 tablet_service.cc:1467] Processing CreateTablet for tablet 9727c0ccc99e478eb0c34096f44de1a6 (DEFAULT_TABLE table=txn_manager_fallback [id=221d15de79984a85af5c16c6e92777a4]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
15:02:52.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.922536 20694 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 9727c0ccc99e478eb0c34096f44de1a6. 1 dirs total, 0 dirs full, 0 dirs failed
15:02:52.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.915869 20549 tablet_service.cc:1467] Processing CreateTablet for tablet 9727c0ccc99e478eb0c34096f44de1a6 (DEFAULT_TABLE table=txn_manager_fallback [id=221d15de79984a85af5c16c6e92777a4]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
15:02:52.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.927412 20549 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 9727c0ccc99e478eb0c34096f44de1a6. 1 dirs total, 0 dirs full, 0 dirs failed
15:02:52.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.928681 20931 tablet_bootstrap.cc:492] T fb15a93555ef4c018b3028f224e8e83b P 1ab780ab6f274ded9a5fdac3c28998c7: Bootstrap starting.
15:02:52.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.933246 20931 tablet_bootstrap.cc:654] T fb15a93555ef4c018b3028f224e8e83b P 1ab780ab6f274ded9a5fdac3c28998c7: Neither blocks nor log segments found. Creating new log.
15:02:52.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.934598 20931 log.cc:826] T fb15a93555ef4c018b3028f224e8e83b P 1ab780ab6f274ded9a5fdac3c28998c7: Log is configured to *not* fsync() on all Append() calls
15:02:52.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.938632 20931 tablet_bootstrap.cc:492] T fb15a93555ef4c018b3028f224e8e83b P 1ab780ab6f274ded9a5fdac3c28998c7: No bootstrap required, opened a new log
15:02:52.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.939273 20931 ts_tablet_manager.cc:1397] T fb15a93555ef4c018b3028f224e8e83b P 1ab780ab6f274ded9a5fdac3c28998c7: Time spent bootstrapping tablet: real 0.011s user 0.005s sys 0.004s
15:02:52.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.943564 20932 tablet_bootstrap.cc:492] T fb15a93555ef4c018b3028f224e8e83b P c45c25ab0a664200896372dcc7565f59: Bootstrap starting.
15:02:52.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.942427 20934 tablet_bootstrap.cc:492] T fb15a93555ef4c018b3028f224e8e83b P a0258fb56e9d4b858dfa55326a152559: Bootstrap starting.
15:02:52.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.948431 20934 tablet_bootstrap.cc:654] T fb15a93555ef4c018b3028f224e8e83b P a0258fb56e9d4b858dfa55326a152559: Neither blocks nor log segments found. Creating new log.
15:02:52.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.949563 20934 log.cc:826] T fb15a93555ef4c018b3028f224e8e83b P a0258fb56e9d4b858dfa55326a152559: Log is configured to *not* fsync() on all Append() calls
15:02:52.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.953583 20931 raft_consensus.cc:357] T fb15a93555ef4c018b3028f224e8e83b P 1ab780ab6f274ded9a5fdac3c28998c7 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c45c25ab0a664200896372dcc7565f59" member_type: VOTER last_known_addr { host: "127.19.180.3" port: 44881 } } peers { permanent_uuid: "a0258fb56e9d4b858dfa55326a152559" member_type: VOTER last_known_addr { host: "127.19.180.1" port: 46735 } } peers { permanent_uuid: "1ab780ab6f274ded9a5fdac3c28998c7" member_type: VOTER last_known_addr { host: "127.19.180.2" port: 35451 } }
15:02:52.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.954444 20932 tablet_bootstrap.cc:654] T fb15a93555ef4c018b3028f224e8e83b P c45c25ab0a664200896372dcc7565f59: Neither blocks nor log segments found. Creating new log.
15:02:52.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.955703 20932 log.cc:826] T fb15a93555ef4c018b3028f224e8e83b P c45c25ab0a664200896372dcc7565f59: Log is configured to *not* fsync() on all Append() calls
15:02:52.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.956353 20931 raft_consensus.cc:383] T fb15a93555ef4c018b3028f224e8e83b P 1ab780ab6f274ded9a5fdac3c28998c7 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:02:52.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.956569 20931 raft_consensus.cc:738] T fb15a93555ef4c018b3028f224e8e83b P 1ab780ab6f274ded9a5fdac3c28998c7 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1ab780ab6f274ded9a5fdac3c28998c7, State: Initialized, Role: FOLLOWER
15:02:52.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.957279 20931 consensus_queue.cc:260] T fb15a93555ef4c018b3028f224e8e83b P 1ab780ab6f274ded9a5fdac3c28998c7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c45c25ab0a664200896372dcc7565f59" member_type: VOTER last_known_addr { host: "127.19.180.3" port: 44881 } } peers { permanent_uuid: "a0258fb56e9d4b858dfa55326a152559" member_type: VOTER last_known_addr { host: "127.19.180.1" port: 46735 } } peers { permanent_uuid: "1ab780ab6f274ded9a5fdac3c28998c7" member_type: VOTER last_known_addr { host: "127.19.180.2" port: 35451 } }
15:02:52.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.959607 20765 heartbeater.cc:502] Master 127.19.180.61:37725 was elected leader, sending a full tablet report...
15:02:52.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.959901 20932 tablet_bootstrap.cc:492] T fb15a93555ef4c018b3028f224e8e83b P c45c25ab0a664200896372dcc7565f59: No bootstrap required, opened a new log
15:02:52.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.960255 20932 ts_tablet_manager.cc:1397] T fb15a93555ef4c018b3028f224e8e83b P c45c25ab0a664200896372dcc7565f59: Time spent bootstrapping tablet: real 0.017s user 0.004s sys 0.003s
15:02:52.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.960502 20931 ts_tablet_manager.cc:1428] T fb15a93555ef4c018b3028f224e8e83b P 1ab780ab6f274ded9a5fdac3c28998c7: Time spent starting tablet: real 0.021s user 0.008s sys 0.001s
15:02:52.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.960573 20934 tablet_bootstrap.cc:492] T fb15a93555ef4c018b3028f224e8e83b P a0258fb56e9d4b858dfa55326a152559: No bootstrap required, opened a new log
15:02:52.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.960834 20934 ts_tablet_manager.cc:1397] T fb15a93555ef4c018b3028f224e8e83b P a0258fb56e9d4b858dfa55326a152559: Time spent bootstrapping tablet: real 0.019s user 0.006s sys 0.000s
15:02:52.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.960875 20931 tablet_bootstrap.cc:492] T 9727c0ccc99e478eb0c34096f44de1a6 P 1ab780ab6f274ded9a5fdac3c28998c7: Bootstrap starting.
15:02:52.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.962507 20931 tablet_bootstrap.cc:654] T 9727c0ccc99e478eb0c34096f44de1a6 P 1ab780ab6f274ded9a5fdac3c28998c7: Neither blocks nor log segments found. Creating new log.
15:02:52.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.969570 20932 raft_consensus.cc:357] T fb15a93555ef4c018b3028f224e8e83b P c45c25ab0a664200896372dcc7565f59 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c45c25ab0a664200896372dcc7565f59" member_type: VOTER last_known_addr { host: "127.19.180.3" port: 44881 } } peers { permanent_uuid: "a0258fb56e9d4b858dfa55326a152559" member_type: VOTER last_known_addr { host: "127.19.180.1" port: 46735 } } peers { permanent_uuid: "1ab780ab6f274ded9a5fdac3c28998c7" member_type: VOTER last_known_addr { host: "127.19.180.2" port: 35451 } }
15:02:52.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.970153 20932 raft_consensus.cc:383] T fb15a93555ef4c018b3028f224e8e83b P c45c25ab0a664200896372dcc7565f59 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:02:52.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.970348 20932 raft_consensus.cc:738] T fb15a93555ef4c018b3028f224e8e83b P c45c25ab0a664200896372dcc7565f59 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c45c25ab0a664200896372dcc7565f59, State: Initialized, Role: FOLLOWER
15:02:52.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.970641 20931 tablet_bootstrap.cc:492] T 9727c0ccc99e478eb0c34096f44de1a6 P 1ab780ab6f274ded9a5fdac3c28998c7: No bootstrap required, opened a new log
15:02:52.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.970790 20931 ts_tablet_manager.cc:1397] T 9727c0ccc99e478eb0c34096f44de1a6 P 1ab780ab6f274ded9a5fdac3c28998c7: Time spent bootstrapping tablet: real 0.010s user 0.004s sys 0.005s
15:02:52.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.971370 20932 consensus_queue.cc:260] T fb15a93555ef4c018b3028f224e8e83b P c45c25ab0a664200896372dcc7565f59 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c45c25ab0a664200896372dcc7565f59" member_type: VOTER last_known_addr { host: "127.19.180.3" port: 44881 } } peers { permanent_uuid: "a0258fb56e9d4b858dfa55326a152559" member_type: VOTER last_known_addr { host: "127.19.180.1" port: 46735 } } peers { permanent_uuid: "1ab780ab6f274ded9a5fdac3c28998c7" member_type: VOTER last_known_addr { host: "127.19.180.2" port: 35451 } }
15:02:52.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.971594 20931 raft_consensus.cc:357] T 9727c0ccc99e478eb0c34096f44de1a6 P 1ab780ab6f274ded9a5fdac3c28998c7 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0258fb56e9d4b858dfa55326a152559" member_type: VOTER last_known_addr { host: "127.19.180.1" port: 46735 } } peers { permanent_uuid: "1ab780ab6f274ded9a5fdac3c28998c7" member_type: VOTER last_known_addr { host: "127.19.180.2" port: 35451 } } peers { permanent_uuid: "c45c25ab0a664200896372dcc7565f59" member_type: VOTER last_known_addr { host: "127.19.180.3" port: 44881 } }
15:02:52.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.971804 20931 raft_consensus.cc:383] T 9727c0ccc99e478eb0c34096f44de1a6 P 1ab780ab6f274ded9a5fdac3c28998c7 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:02:52.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.971896 20931 raft_consensus.cc:738] T 9727c0ccc99e478eb0c34096f44de1a6 P 1ab780ab6f274ded9a5fdac3c28998c7 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1ab780ab6f274ded9a5fdac3c28998c7, State: Initialized, Role: FOLLOWER
15:02:52.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.972126 20931 consensus_queue.cc:260] T 9727c0ccc99e478eb0c34096f44de1a6 P 1ab780ab6f274ded9a5fdac3c28998c7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0258fb56e9d4b858dfa55326a152559" member_type: VOTER last_known_addr { host: "127.19.180.1" port: 46735 } } peers { permanent_uuid: "1ab780ab6f274ded9a5fdac3c28998c7" member_type: VOTER last_known_addr { host: "127.19.180.2" port: 35451 } } peers { permanent_uuid: "c45c25ab0a664200896372dcc7565f59" member_type: VOTER last_known_addr { host: "127.19.180.3" port: 44881 } }
15:02:52.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.972760 20931 ts_tablet_manager.cc:1428] T 9727c0ccc99e478eb0c34096f44de1a6 P 1ab780ab6f274ded9a5fdac3c28998c7: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
15:02:52.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.973605 20906 heartbeater.cc:502] Master 127.19.180.61:37725 was elected leader, sending a full tablet report...
15:02:52.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.976029 20932 ts_tablet_manager.cc:1428] T fb15a93555ef4c018b3028f224e8e83b P c45c25ab0a664200896372dcc7565f59: Time spent starting tablet: real 0.016s user 0.007s sys 0.004s
15:02:52.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.976537 20932 tablet_bootstrap.cc:492] T 9727c0ccc99e478eb0c34096f44de1a6 P c45c25ab0a664200896372dcc7565f59: Bootstrap starting.
15:02:52.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.976516 20934 raft_consensus.cc:357] T fb15a93555ef4c018b3028f224e8e83b P a0258fb56e9d4b858dfa55326a152559 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c45c25ab0a664200896372dcc7565f59" member_type: VOTER last_known_addr { host: "127.19.180.3" port: 44881 } } peers { permanent_uuid: "a0258fb56e9d4b858dfa55326a152559" member_type: VOTER last_known_addr { host: "127.19.180.1" port: 46735 } } peers { permanent_uuid: "1ab780ab6f274ded9a5fdac3c28998c7" member_type: VOTER last_known_addr { host: "127.19.180.2" port: 35451 } }
15:02:52.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.976949 20934 raft_consensus.cc:383] T fb15a93555ef4c018b3028f224e8e83b P a0258fb56e9d4b858dfa55326a152559 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:02:52.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.977059 20934 raft_consensus.cc:738] T fb15a93555ef4c018b3028f224e8e83b P a0258fb56e9d4b858dfa55326a152559 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a0258fb56e9d4b858dfa55326a152559, State: Initialized, Role: FOLLOWER
15:02:52.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.977757 20934 consensus_queue.cc:260] T fb15a93555ef4c018b3028f224e8e83b P a0258fb56e9d4b858dfa55326a152559 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c45c25ab0a664200896372dcc7565f59" member_type: VOTER last_known_addr { host: "127.19.180.3" port: 44881 } } peers { permanent_uuid: "a0258fb56e9d4b858dfa55326a152559" member_type: VOTER last_known_addr { host: "127.19.180.1" port: 46735 } } peers { permanent_uuid: "1ab780ab6f274ded9a5fdac3c28998c7" member_type: VOTER last_known_addr { host: "127.19.180.2" port: 35451 } }
15:02:52.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.978742 20932 tablet_bootstrap.cc:654] T 9727c0ccc99e478eb0c34096f44de1a6 P c45c25ab0a664200896372dcc7565f59: Neither blocks nor log segments found. Creating new log.
15:02:52.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.981153 20932 tablet_bootstrap.cc:492] T 9727c0ccc99e478eb0c34096f44de1a6 P c45c25ab0a664200896372dcc7565f59: No bootstrap required, opened a new log
15:02:52.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.981300 20932 ts_tablet_manager.cc:1397] T 9727c0ccc99e478eb0c34096f44de1a6 P c45c25ab0a664200896372dcc7565f59: Time spent bootstrapping tablet: real 0.005s user 0.003s sys 0.001s
15:02:52.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.982162 20932 raft_consensus.cc:357] T 9727c0ccc99e478eb0c34096f44de1a6 P c45c25ab0a664200896372dcc7565f59 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0258fb56e9d4b858dfa55326a152559" member_type: VOTER last_known_addr { host: "127.19.180.1" port: 46735 } } peers { permanent_uuid: "1ab780ab6f274ded9a5fdac3c28998c7" member_type: VOTER last_known_addr { host: "127.19.180.2" port: 35451 } } peers { permanent_uuid: "c45c25ab0a664200896372dcc7565f59" member_type: VOTER last_known_addr { host: "127.19.180.3" port: 44881 } }
15:02:52.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.982348 20932 raft_consensus.cc:383] T 9727c0ccc99e478eb0c34096f44de1a6 P c45c25ab0a664200896372dcc7565f59 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:02:52.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.982432 20932 raft_consensus.cc:738] T 9727c0ccc99e478eb0c34096f44de1a6 P c45c25ab0a664200896372dcc7565f59 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c45c25ab0a664200896372dcc7565f59, State: Initialized, Role: FOLLOWER
15:02:52.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.982652 20932 consensus_queue.cc:260] T 9727c0ccc99e478eb0c34096f44de1a6 P c45c25ab0a664200896372dcc7565f59 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0258fb56e9d4b858dfa55326a152559" member_type: VOTER last_known_addr { host: "127.19.180.1" port: 46735 } } peers { permanent_uuid: "1ab780ab6f274ded9a5fdac3c28998c7" member_type: VOTER last_known_addr { host: "127.19.180.2" port: 35451 } } peers { permanent_uuid: "c45c25ab0a664200896372dcc7565f59" member_type: VOTER last_known_addr { host: "127.19.180.3" port: 44881 } }
15:02:52.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.983150 20932 ts_tablet_manager.cc:1428] T 9727c0ccc99e478eb0c34096f44de1a6 P c45c25ab0a664200896372dcc7565f59: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
15:02:52.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.990981 20934 ts_tablet_manager.cc:1428] T fb15a93555ef4c018b3028f224e8e83b P a0258fb56e9d4b858dfa55326a152559: Time spent starting tablet: real 0.030s user 0.010s sys 0.005s
15:02:52.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.991405 20934 tablet_bootstrap.cc:492] T 9727c0ccc99e478eb0c34096f44de1a6 P a0258fb56e9d4b858dfa55326a152559: Bootstrap starting.
15:02:52.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.993105 20934 tablet_bootstrap.cc:654] T 9727c0ccc99e478eb0c34096f44de1a6 P a0258fb56e9d4b858dfa55326a152559: Neither blocks nor log segments found. Creating new log.
15:02:52.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.995167 20934 tablet_bootstrap.cc:492] T 9727c0ccc99e478eb0c34096f44de1a6 P a0258fb56e9d4b858dfa55326a152559: No bootstrap required, opened a new log
15:02:52.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.995307 20934 ts_tablet_manager.cc:1397] T 9727c0ccc99e478eb0c34096f44de1a6 P a0258fb56e9d4b858dfa55326a152559: Time spent bootstrapping tablet: real 0.004s user 0.000s sys 0.003s
15:02:52.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.996045 20934 raft_consensus.cc:357] T 9727c0ccc99e478eb0c34096f44de1a6 P a0258fb56e9d4b858dfa55326a152559 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0258fb56e9d4b858dfa55326a152559" member_type: VOTER last_known_addr { host: "127.19.180.1" port: 46735 } } peers { permanent_uuid: "1ab780ab6f274ded9a5fdac3c28998c7" member_type: VOTER last_known_addr { host: "127.19.180.2" port: 35451 } } peers { permanent_uuid: "c45c25ab0a664200896372dcc7565f59" member_type: VOTER last_known_addr { host: "127.19.180.3" port: 44881 } }
15:02:52.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.996248 20934 raft_consensus.cc:383] T 9727c0ccc99e478eb0c34096f44de1a6 P a0258fb56e9d4b858dfa55326a152559 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:02:52.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.996335 20934 raft_consensus.cc:738] T 9727c0ccc99e478eb0c34096f44de1a6 P a0258fb56e9d4b858dfa55326a152559 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a0258fb56e9d4b858dfa55326a152559, State: Initialized, Role: FOLLOWER
15:02:52.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.996551 20934 consensus_queue.cc:260] T 9727c0ccc99e478eb0c34096f44de1a6 P a0258fb56e9d4b858dfa55326a152559 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0258fb56e9d4b858dfa55326a152559" member_type: VOTER last_known_addr { host: "127.19.180.1" port: 46735 } } peers { permanent_uuid: "1ab780ab6f274ded9a5fdac3c28998c7" member_type: VOTER last_known_addr { host: "127.19.180.2" port: 35451 } } peers { permanent_uuid: "c45c25ab0a664200896372dcc7565f59" member_type: VOTER last_known_addr { host: "127.19.180.3" port: 44881 } }
15:02:52.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:52.997053 20934 ts_tablet_manager.cc:1428] T 9727c0ccc99e478eb0c34096f44de1a6 P a0258fb56e9d4b858dfa55326a152559: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
15:02:53.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.026073 20940 raft_consensus.cc:491] T fb15a93555ef4c018b3028f224e8e83b P a0258fb56e9d4b858dfa55326a152559 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:02:53.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.026342 20940 raft_consensus.cc:513] T fb15a93555ef4c018b3028f224e8e83b P a0258fb56e9d4b858dfa55326a152559 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c45c25ab0a664200896372dcc7565f59" member_type: VOTER last_known_addr { host: "127.19.180.3" port: 44881 } } peers { permanent_uuid: "a0258fb56e9d4b858dfa55326a152559" member_type: VOTER last_known_addr { host: "127.19.180.1" port: 46735 } } peers { permanent_uuid: "1ab780ab6f274ded9a5fdac3c28998c7" member_type: VOTER last_known_addr { host: "127.19.180.2" port: 35451 } }
15:02:53.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.028259 20940 leader_election.cc:290] T fb15a93555ef4c018b3028f224e8e83b P a0258fb56e9d4b858dfa55326a152559 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c45c25ab0a664200896372dcc7565f59 (127.19.180.3:44881), 1ab780ab6f274ded9a5fdac3c28998c7 (127.19.180.2:35451)
15:02:53.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.034924 20718 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fb15a93555ef4c018b3028f224e8e83b" candidate_uuid: "a0258fb56e9d4b858dfa55326a152559" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1ab780ab6f274ded9a5fdac3c28998c7" is_pre_election: true
15:02:53.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.034922 20859 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fb15a93555ef4c018b3028f224e8e83b" candidate_uuid: "a0258fb56e9d4b858dfa55326a152559" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c45c25ab0a664200896372dcc7565f59" is_pre_election: true
15:02:53.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.035389 20718 raft_consensus.cc:2463] T fb15a93555ef4c018b3028f224e8e83b P 1ab780ab6f274ded9a5fdac3c28998c7 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a0258fb56e9d4b858dfa55326a152559 in term 0.
15:02:53.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.035419 20859 raft_consensus.cc:2463] T fb15a93555ef4c018b3028f224e8e83b P c45c25ab0a664200896372dcc7565f59 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a0258fb56e9d4b858dfa55326a152559 in term 0.
15:02:53.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.035889 20499 leader_election.cc:304] T fb15a93555ef4c018b3028f224e8e83b P a0258fb56e9d4b858dfa55326a152559 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1ab780ab6f274ded9a5fdac3c28998c7, a0258fb56e9d4b858dfa55326a152559; no voters:
15:02:53.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.036211 20940 raft_consensus.cc:2798] T fb15a93555ef4c018b3028f224e8e83b P a0258fb56e9d4b858dfa55326a152559 [term 0 FOLLOWER]: Leader pre-election won for term 1
15:02:53.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.036332 20940 raft_consensus.cc:491] T fb15a93555ef4c018b3028f224e8e83b P a0258fb56e9d4b858dfa55326a152559 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
15:02:53.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.036393 20940 raft_consensus.cc:3054] T fb15a93555ef4c018b3028f224e8e83b P a0258fb56e9d4b858dfa55326a152559 [term 0 FOLLOWER]: Advancing to term 1
15:02:53.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.038327 20940 raft_consensus.cc:513] T fb15a93555ef4c018b3028f224e8e83b P a0258fb56e9d4b858dfa55326a152559 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c45c25ab0a664200896372dcc7565f59" member_type: VOTER last_known_addr { host: "127.19.180.3" port: 44881 } } peers { permanent_uuid: "a0258fb56e9d4b858dfa55326a152559" member_type: VOTER last_known_addr { host: "127.19.180.1" port: 46735 } } peers { permanent_uuid: "1ab780ab6f274ded9a5fdac3c28998c7" member_type: VOTER last_known_addr { host: "127.19.180.2" port: 35451 } }
15:02:53.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.038892 20940 leader_election.cc:290] T fb15a93555ef4c018b3028f224e8e83b P a0258fb56e9d4b858dfa55326a152559 [CANDIDATE]: Term 1 election: Requested vote from peers c45c25ab0a664200896372dcc7565f59 (127.19.180.3:44881), 1ab780ab6f274ded9a5fdac3c28998c7 (127.19.180.2:35451)
15:02:53.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.039283 20859 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fb15a93555ef4c018b3028f224e8e83b" candidate_uuid: "a0258fb56e9d4b858dfa55326a152559" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c45c25ab0a664200896372dcc7565f59"
15:02:53.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.039294 20718 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fb15a93555ef4c018b3028f224e8e83b" candidate_uuid: "a0258fb56e9d4b858dfa55326a152559" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1ab780ab6f274ded9a5fdac3c28998c7"
15:02:53.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.039443 20718 raft_consensus.cc:3054] T fb15a93555ef4c018b3028f224e8e83b P 1ab780ab6f274ded9a5fdac3c28998c7 [term 0 FOLLOWER]: Advancing to term 1
15:02:53.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.039455 20859 raft_consensus.cc:3054] T fb15a93555ef4c018b3028f224e8e83b P c45c25ab0a664200896372dcc7565f59 [term 0 FOLLOWER]: Advancing to term 1
15:02:53.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.041332 20718 raft_consensus.cc:2463] T fb15a93555ef4c018b3028f224e8e83b P 1ab780ab6f274ded9a5fdac3c28998c7 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a0258fb56e9d4b858dfa55326a152559 in term 1.
15:02:53.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.041443 20859 raft_consensus.cc:2463] T fb15a93555ef4c018b3028f224e8e83b P c45c25ab0a664200896372dcc7565f59 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a0258fb56e9d4b858dfa55326a152559 in term 1.
15:02:53.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.041677 20499 leader_election.cc:304] T fb15a93555ef4c018b3028f224e8e83b P a0258fb56e9d4b858dfa55326a152559 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1ab780ab6f274ded9a5fdac3c28998c7, a0258fb56e9d4b858dfa55326a152559; no voters:
15:02:53.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.041962 20940 raft_consensus.cc:2798] T fb15a93555ef4c018b3028f224e8e83b P a0258fb56e9d4b858dfa55326a152559 [term 1 FOLLOWER]: Leader election won for term 1
15:02:53.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.042372 20940 raft_consensus.cc:695] T fb15a93555ef4c018b3028f224e8e83b P a0258fb56e9d4b858dfa55326a152559 [term 1 LEADER]: Becoming Leader. State: Replica: a0258fb56e9d4b858dfa55326a152559, State: Running, Role: LEADER
15:02:53.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:53.042698 20907 tablet.cc:2367] T 9727c0ccc99e478eb0c34096f44de1a6 P c45c25ab0a664200896372dcc7565f59: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:02:53.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.042879 20940 consensus_queue.cc:237] T fb15a93555ef4c018b3028f224e8e83b P a0258fb56e9d4b858dfa55326a152559 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c45c25ab0a664200896372dcc7565f59" member_type: VOTER last_known_addr { host: "127.19.180.3" port: 44881 } } peers { permanent_uuid: "a0258fb56e9d4b858dfa55326a152559" member_type: VOTER last_known_addr { host: "127.19.180.1" port: 46735 } } peers { permanent_uuid: "1ab780ab6f274ded9a5fdac3c28998c7" member_type: VOTER last_known_addr { host: "127.19.180.2" port: 35451 } }
15:02:53.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.048363 20297 catalog_manager.cc:5526] T fb15a93555ef4c018b3028f224e8e83b P a0258fb56e9d4b858dfa55326a152559 reported cstate change: term changed from 0 to 1, leader changed from <none> to a0258fb56e9d4b858dfa55326a152559 (127.19.180.1). New cstate: current_term: 1 leader_uuid: "a0258fb56e9d4b858dfa55326a152559" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c45c25ab0a664200896372dcc7565f59" member_type: VOTER last_known_addr { host: "127.19.180.3" port: 44881 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a0258fb56e9d4b858dfa55326a152559" member_type: VOTER last_known_addr { host: "127.19.180.1" port: 46735 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1ab780ab6f274ded9a5fdac3c28998c7" member_type: VOTER last_known_addr { host: "127.19.180.2" port: 35451 } health_report { overall_health: UNKNOWN } } }
15:02:53.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:53.080066 20766 tablet.cc:2367] T 9727c0ccc99e478eb0c34096f44de1a6 P 1ab780ab6f274ded9a5fdac3c28998c7: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:02:53.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:53.122416 20625 tablet.cc:2367] T 9727c0ccc99e478eb0c34096f44de1a6 P a0258fb56e9d4b858dfa55326a152559: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:02:53.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.153414 20939 raft_consensus.cc:491] T 9727c0ccc99e478eb0c34096f44de1a6 P c45c25ab0a664200896372dcc7565f59 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:02:53.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.153612 20939 raft_consensus.cc:513] T 9727c0ccc99e478eb0c34096f44de1a6 P c45c25ab0a664200896372dcc7565f59 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0258fb56e9d4b858dfa55326a152559" member_type: VOTER last_known_addr { host: "127.19.180.1" port: 46735 } } peers { permanent_uuid: "1ab780ab6f274ded9a5fdac3c28998c7" member_type: VOTER last_known_addr { host: "127.19.180.2" port: 35451 } } peers { permanent_uuid: "c45c25ab0a664200896372dcc7565f59" member_type: VOTER last_known_addr { host: "127.19.180.3" port: 44881 } }
15:02:53.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.155239 20939 leader_election.cc:290] T 9727c0ccc99e478eb0c34096f44de1a6 P c45c25ab0a664200896372dcc7565f59 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a0258fb56e9d4b858dfa55326a152559 (127.19.180.1:46735), 1ab780ab6f274ded9a5fdac3c28998c7 (127.19.180.2:35451)
15:02:53.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.160872 20718 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9727c0ccc99e478eb0c34096f44de1a6" candidate_uuid: "c45c25ab0a664200896372dcc7565f59" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1ab780ab6f274ded9a5fdac3c28998c7" is_pre_election: true
15:02:53.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.161105 20718 raft_consensus.cc:2463] T 9727c0ccc99e478eb0c34096f44de1a6 P 1ab780ab6f274ded9a5fdac3c28998c7 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c45c25ab0a664200896372dcc7565f59 in term 0.
15:02:53.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.161121 20576 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9727c0ccc99e478eb0c34096f44de1a6" candidate_uuid: "c45c25ab0a664200896372dcc7565f59" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a0258fb56e9d4b858dfa55326a152559" is_pre_election: true
15:02:53.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.161518 20576 raft_consensus.cc:2463] T 9727c0ccc99e478eb0c34096f44de1a6 P a0258fb56e9d4b858dfa55326a152559 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c45c25ab0a664200896372dcc7565f59 in term 0.
15:02:53.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.161537 20783 leader_election.cc:304] T 9727c0ccc99e478eb0c34096f44de1a6 P c45c25ab0a664200896372dcc7565f59 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1ab780ab6f274ded9a5fdac3c28998c7, c45c25ab0a664200896372dcc7565f59; no voters:
15:02:53.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.161891 20939 raft_consensus.cc:2798] T 9727c0ccc99e478eb0c34096f44de1a6 P c45c25ab0a664200896372dcc7565f59 [term 0 FOLLOWER]: Leader pre-election won for term 1
15:02:53.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.162025 20939 raft_consensus.cc:491] T 9727c0ccc99e478eb0c34096f44de1a6 P c45c25ab0a664200896372dcc7565f59 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
15:02:53.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.162127 20939 raft_consensus.cc:3054] T 9727c0ccc99e478eb0c34096f44de1a6 P c45c25ab0a664200896372dcc7565f59 [term 0 FOLLOWER]: Advancing to term 1
15:02:53.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.164093 20939 raft_consensus.cc:513] T 9727c0ccc99e478eb0c34096f44de1a6 P c45c25ab0a664200896372dcc7565f59 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0258fb56e9d4b858dfa55326a152559" member_type: VOTER last_known_addr { host: "127.19.180.1" port: 46735 } } peers { permanent_uuid: "1ab780ab6f274ded9a5fdac3c28998c7" member_type: VOTER last_known_addr { host: "127.19.180.2" port: 35451 } } peers { permanent_uuid: "c45c25ab0a664200896372dcc7565f59" member_type: VOTER last_known_addr { host: "127.19.180.3" port: 44881 } }
15:02:53.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.164723 20939 leader_election.cc:290] T 9727c0ccc99e478eb0c34096f44de1a6 P c45c25ab0a664200896372dcc7565f59 [CANDIDATE]: Term 1 election: Requested vote from peers a0258fb56e9d4b858dfa55326a152559 (127.19.180.1:46735), 1ab780ab6f274ded9a5fdac3c28998c7 (127.19.180.2:35451)
15:02:53.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.165025 20576 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9727c0ccc99e478eb0c34096f44de1a6" candidate_uuid: "c45c25ab0a664200896372dcc7565f59" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a0258fb56e9d4b858dfa55326a152559"
15:02:53.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.165141 20718 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9727c0ccc99e478eb0c34096f44de1a6" candidate_uuid: "c45c25ab0a664200896372dcc7565f59" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1ab780ab6f274ded9a5fdac3c28998c7"
15:02:53.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.165194 20576 raft_consensus.cc:3054] T 9727c0ccc99e478eb0c34096f44de1a6 P a0258fb56e9d4b858dfa55326a152559 [term 0 FOLLOWER]: Advancing to term 1
15:02:53.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.165338 20718 raft_consensus.cc:3054] T 9727c0ccc99e478eb0c34096f44de1a6 P 1ab780ab6f274ded9a5fdac3c28998c7 [term 0 FOLLOWER]: Advancing to term 1
15:02:53.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.167140 20718 raft_consensus.cc:2463] T 9727c0ccc99e478eb0c34096f44de1a6 P 1ab780ab6f274ded9a5fdac3c28998c7 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c45c25ab0a664200896372dcc7565f59 in term 1.
15:02:53.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.167145 20576 raft_consensus.cc:2463] T 9727c0ccc99e478eb0c34096f44de1a6 P a0258fb56e9d4b858dfa55326a152559 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c45c25ab0a664200896372dcc7565f59 in term 1.
15:02:53.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.167513 20783 leader_election.cc:304] T 9727c0ccc99e478eb0c34096f44de1a6 P c45c25ab0a664200896372dcc7565f59 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a0258fb56e9d4b858dfa55326a152559, c45c25ab0a664200896372dcc7565f59; no voters:
15:02:53.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.167814 20939 raft_consensus.cc:2798] T 9727c0ccc99e478eb0c34096f44de1a6 P c45c25ab0a664200896372dcc7565f59 [term 1 FOLLOWER]: Leader election won for term 1
15:02:53.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.168296 20939 raft_consensus.cc:695] T 9727c0ccc99e478eb0c34096f44de1a6 P c45c25ab0a664200896372dcc7565f59 [term 1 LEADER]: Becoming Leader. State: Replica: c45c25ab0a664200896372dcc7565f59, State: Running, Role: LEADER
15:02:53.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.168807 20939 consensus_queue.cc:237] T 9727c0ccc99e478eb0c34096f44de1a6 P c45c25ab0a664200896372dcc7565f59 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0258fb56e9d4b858dfa55326a152559" member_type: VOTER last_known_addr { host: "127.19.180.1" port: 46735 } } peers { permanent_uuid: "1ab780ab6f274ded9a5fdac3c28998c7" member_type: VOTER last_known_addr { host: "127.19.180.2" port: 35451 } } peers { permanent_uuid: "c45c25ab0a664200896372dcc7565f59" member_type: VOTER last_known_addr { host: "127.19.180.3" port: 44881 } }
15:02:53.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.173815 20297 catalog_manager.cc:5526] T 9727c0ccc99e478eb0c34096f44de1a6 P c45c25ab0a664200896372dcc7565f59 reported cstate change: term changed from 0 to 1, leader changed from <none> to c45c25ab0a664200896372dcc7565f59 (127.19.180.3). New cstate: current_term: 1 leader_uuid: "c45c25ab0a664200896372dcc7565f59" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0258fb56e9d4b858dfa55326a152559" member_type: VOTER last_known_addr { host: "127.19.180.1" port: 46735 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1ab780ab6f274ded9a5fdac3c28998c7" member_type: VOTER last_known_addr { host: "127.19.180.2" port: 35451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c45c25ab0a664200896372dcc7565f59" member_type: VOTER last_known_addr { host: "127.19.180.3" port: 44881 } health_report { overall_health: HEALTHY } } }
15:02:53.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.266198 20297 catalog_manager.cc:1909] Servicing CreateTable request from {username='slave'} at 127.0.0.1:37796:
15:02:53.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
15:02:53.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
15:02:53.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:02:53.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
15:02:53.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
15:02:53.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
15:02:53.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
15:02:53.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
15:02:53.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
15:02:53.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
15:02:53.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
15:02:53.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:02:53.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:02:53.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
15:02:53.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
15:02:53.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
15:02:53.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
15:02:53.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
15:02:53.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
15:02:53.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
15:02:53.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
15:02:53.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:02:53.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:02:53.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
15:02:53.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
15:02:53.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
15:02:53.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
15:02:53.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
15:02:53.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
15:02:53.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
15:02:53.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
15:02:53.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:02:53.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:02:53.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
15:02:53.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
15:02:53.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
15:02:53.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
15:02:53.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
15:02:53.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
15:02:53.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
15:02:53.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
15:02:53.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:02:53.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:02:53.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
15:02:53.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
15:02:53.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
15:02:53.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
15:02:53.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:02:53.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
15:02:53.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
15:02:53.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:02:53.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
15:02:53.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:02:53.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:02:53.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:02:53.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
15:02:53.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:53.267192 20297 catalog_manager.cc:6885] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
15:02:53.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.278726 20836 tablet_service.cc:1467] Processing CreateTablet for tablet 0962a0a9b1f842b08b040f6e6a3c050e (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=48b774e56c0a48579a937d7b974ec75b]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
15:02:53.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.279736 20836 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0962a0a9b1f842b08b040f6e6a3c050e. 1 dirs total, 0 dirs full, 0 dirs failed
15:02:53.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.279875 20694 tablet_service.cc:1467] Processing CreateTablet for tablet 0962a0a9b1f842b08b040f6e6a3c050e (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=48b774e56c0a48579a937d7b974ec75b]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
15:02:53.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.280687 20694 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0962a0a9b1f842b08b040f6e6a3c050e. 1 dirs total, 0 dirs full, 0 dirs failed
15:02:53.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.278784 20551 tablet_service.cc:1467] Processing CreateTablet for tablet 0962a0a9b1f842b08b040f6e6a3c050e (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=48b774e56c0a48579a937d7b974ec75b]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
15:02:53.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.286028 20551 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0962a0a9b1f842b08b040f6e6a3c050e. 1 dirs total, 0 dirs full, 0 dirs failed
15:02:53.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.286505 20932 tablet_bootstrap.cc:492] T 0962a0a9b1f842b08b040f6e6a3c050e P c45c25ab0a664200896372dcc7565f59: Bootstrap starting.
15:02:53.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.286993 20931 tablet_bootstrap.cc:492] T 0962a0a9b1f842b08b040f6e6a3c050e P 1ab780ab6f274ded9a5fdac3c28998c7: Bootstrap starting.
15:02:53.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.288043 20932 tablet_bootstrap.cc:654] T 0962a0a9b1f842b08b040f6e6a3c050e P c45c25ab0a664200896372dcc7565f59: Neither blocks nor log segments found. Creating new log.
15:02:53.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.288578 20931 tablet_bootstrap.cc:654] T 0962a0a9b1f842b08b040f6e6a3c050e P 1ab780ab6f274ded9a5fdac3c28998c7: Neither blocks nor log segments found. Creating new log.
15:02:53.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.289836 20932 tablet_bootstrap.cc:492] T 0962a0a9b1f842b08b040f6e6a3c050e P c45c25ab0a664200896372dcc7565f59: No bootstrap required, opened a new log
15:02:53.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.289983 20932 ts_tablet_manager.cc:1397] T 0962a0a9b1f842b08b040f6e6a3c050e P c45c25ab0a664200896372dcc7565f59: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
15:02:53.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.290658 20932 raft_consensus.cc:357] T 0962a0a9b1f842b08b040f6e6a3c050e P c45c25ab0a664200896372dcc7565f59 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0258fb56e9d4b858dfa55326a152559" member_type: VOTER last_known_addr { host: "127.19.180.1" port: 46735 } } peers { permanent_uuid: "1ab780ab6f274ded9a5fdac3c28998c7" member_type: VOTER last_known_addr { host: "127.19.180.2" port: 35451 } } peers { permanent_uuid: "c45c25ab0a664200896372dcc7565f59" member_type: VOTER last_known_addr { host: "127.19.180.3" port: 44881 } }
15:02:53.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.290882 20932 raft_consensus.cc:383] T 0962a0a9b1f842b08b040f6e6a3c050e P c45c25ab0a664200896372dcc7565f59 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:02:53.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.290997 20932 raft_consensus.cc:738] T 0962a0a9b1f842b08b040f6e6a3c050e P c45c25ab0a664200896372dcc7565f59 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c45c25ab0a664200896372dcc7565f59, State: Initialized, Role: FOLLOWER
15:02:53.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.291256 20932 consensus_queue.cc:260] T 0962a0a9b1f842b08b040f6e6a3c050e P c45c25ab0a664200896372dcc7565f59 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0258fb56e9d4b858dfa55326a152559" member_type: VOTER last_known_addr { host: "127.19.180.1" port: 46735 } } peers { permanent_uuid: "1ab780ab6f274ded9a5fdac3c28998c7" member_type: VOTER last_known_addr { host: "127.19.180.2" port: 35451 } } peers { permanent_uuid: "c45c25ab0a664200896372dcc7565f59" member_type: VOTER last_known_addr { host: "127.19.180.3" port: 44881 } }
15:02:53.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.291580 20931 tablet_bootstrap.cc:492] T 0962a0a9b1f842b08b040f6e6a3c050e P 1ab780ab6f274ded9a5fdac3c28998c7: No bootstrap required, opened a new log
15:02:53.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.291676 20932 ts_tablet_manager.cc:1428] T 0962a0a9b1f842b08b040f6e6a3c050e P c45c25ab0a664200896372dcc7565f59: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
15:02:53.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.291752 20931 ts_tablet_manager.cc:1397] T 0962a0a9b1f842b08b040f6e6a3c050e P 1ab780ab6f274ded9a5fdac3c28998c7: Time spent bootstrapping tablet: real 0.005s user 0.002s sys 0.000s
15:02:53.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.292469 20931 raft_consensus.cc:357] T 0962a0a9b1f842b08b040f6e6a3c050e P 1ab780ab6f274ded9a5fdac3c28998c7 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0258fb56e9d4b858dfa55326a152559" member_type: VOTER last_known_addr { host: "127.19.180.1" port: 46735 } } peers { permanent_uuid: "1ab780ab6f274ded9a5fdac3c28998c7" member_type: VOTER last_known_addr { host: "127.19.180.2" port: 35451 } } peers { permanent_uuid: "c45c25ab0a664200896372dcc7565f59" member_type: VOTER last_known_addr { host: "127.19.180.3" port: 44881 } }
15:02:53.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.293196 20939 tablet_replica.cc:440] T 0962a0a9b1f842b08b040f6e6a3c050e P c45c25ab0a664200896372dcc7565f59: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0258fb56e9d4b858dfa55326a152559" member_type: VOTER last_known_addr { host: "127.19.180.1" port: 46735 } } peers { permanent_uuid: "1ab780ab6f274ded9a5fdac3c28998c7" member_type: VOTER last_known_addr { host: "127.19.180.2" port: 35451 } } peers { permanent_uuid: "c45c25ab0a664200896372dcc7565f59" member_type: VOTER last_known_addr { host: "127.19.180.3" port: 44881 } } }
15:02:53.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.293468 20939 tablet_replica.cc:443] T 0962a0a9b1f842b08b040f6e6a3c050e P c45c25ab0a664200896372dcc7565f59: This TxnStatusTablet replica's current role is: FOLLOWER
15:02:53.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.293521 20931 raft_consensus.cc:383] T 0962a0a9b1f842b08b040f6e6a3c050e P 1ab780ab6f274ded9a5fdac3c28998c7 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:02:53.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.293624 20931 raft_consensus.cc:738] T 0962a0a9b1f842b08b040f6e6a3c050e P 1ab780ab6f274ded9a5fdac3c28998c7 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1ab780ab6f274ded9a5fdac3c28998c7, State: Initialized, Role: FOLLOWER
15:02:53.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.293884 20931 consensus_queue.cc:260] T 0962a0a9b1f842b08b040f6e6a3c050e P 1ab780ab6f274ded9a5fdac3c28998c7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0258fb56e9d4b858dfa55326a152559" member_type: VOTER last_known_addr { host: "127.19.180.1" port: 46735 } } peers { permanent_uuid: "1ab780ab6f274ded9a5fdac3c28998c7" member_type: VOTER last_known_addr { host: "127.19.180.2" port: 35451 } } peers { permanent_uuid: "c45c25ab0a664200896372dcc7565f59" member_type: VOTER last_known_addr { host: "127.19.180.3" port: 44881 } }
15:02:53.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.294366 20937 tablet_replica.cc:440] T 0962a0a9b1f842b08b040f6e6a3c050e P 1ab780ab6f274ded9a5fdac3c28998c7: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0258fb56e9d4b858dfa55326a152559" member_type: VOTER last_known_addr { host: "127.19.180.1" port: 46735 } } peers { permanent_uuid: "1ab780ab6f274ded9a5fdac3c28998c7" member_type: VOTER last_known_addr { host: "127.19.180.2" port: 35451 } } peers { permanent_uuid: "c45c25ab0a664200896372dcc7565f59" member_type: VOTER last_known_addr { host: "127.19.180.3" port: 44881 } } }
15:02:53.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.294670 20937 tablet_replica.cc:443] T 0962a0a9b1f842b08b040f6e6a3c050e P 1ab780ab6f274ded9a5fdac3c28998c7: This TxnStatusTablet replica's current role is: FOLLOWER
15:02:53.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.294384 20931 ts_tablet_manager.cc:1428] T 0962a0a9b1f842b08b040f6e6a3c050e P 1ab780ab6f274ded9a5fdac3c28998c7: Time spent starting tablet: real 0.003s user 0.001s sys 0.001s
15:02:53.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.299458 20934 tablet_bootstrap.cc:492] T 0962a0a9b1f842b08b040f6e6a3c050e P a0258fb56e9d4b858dfa55326a152559: Bootstrap starting.
15:02:53.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.301057 20934 tablet_bootstrap.cc:654] T 0962a0a9b1f842b08b040f6e6a3c050e P a0258fb56e9d4b858dfa55326a152559: Neither blocks nor log segments found. Creating new log.
15:02:53.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.302738 20934 tablet_bootstrap.cc:492] T 0962a0a9b1f842b08b040f6e6a3c050e P a0258fb56e9d4b858dfa55326a152559: No bootstrap required, opened a new log
15:02:53.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.302860 20934 ts_tablet_manager.cc:1397] T 0962a0a9b1f842b08b040f6e6a3c050e P a0258fb56e9d4b858dfa55326a152559: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.001s
15:02:53.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.303467 20934 raft_consensus.cc:357] T 0962a0a9b1f842b08b040f6e6a3c050e P a0258fb56e9d4b858dfa55326a152559 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0258fb56e9d4b858dfa55326a152559" member_type: VOTER last_known_addr { host: "127.19.180.1" port: 46735 } } peers { permanent_uuid: "1ab780ab6f274ded9a5fdac3c28998c7" member_type: VOTER last_known_addr { host: "127.19.180.2" port: 35451 } } peers { permanent_uuid: "c45c25ab0a664200896372dcc7565f59" member_type: VOTER last_known_addr { host: "127.19.180.3" port: 44881 } }
15:02:53.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.303632 20934 raft_consensus.cc:383] T 0962a0a9b1f842b08b040f6e6a3c050e P a0258fb56e9d4b858dfa55326a152559 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:02:53.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.303679 20934 raft_consensus.cc:738] T 0962a0a9b1f842b08b040f6e6a3c050e P a0258fb56e9d4b858dfa55326a152559 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a0258fb56e9d4b858dfa55326a152559, State: Initialized, Role: FOLLOWER
15:02:53.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.303872 20934 consensus_queue.cc:260] T 0962a0a9b1f842b08b040f6e6a3c050e P a0258fb56e9d4b858dfa55326a152559 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0258fb56e9d4b858dfa55326a152559" member_type: VOTER last_known_addr { host: "127.19.180.1" port: 46735 } } peers { permanent_uuid: "1ab780ab6f274ded9a5fdac3c28998c7" member_type: VOTER last_known_addr { host: "127.19.180.2" port: 35451 } } peers { permanent_uuid: "c45c25ab0a664200896372dcc7565f59" member_type: VOTER last_known_addr { host: "127.19.180.3" port: 44881 } }
15:02:53.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.304309 20934 ts_tablet_manager.cc:1428] T 0962a0a9b1f842b08b040f6e6a3c050e P a0258fb56e9d4b858dfa55326a152559: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
15:02:53.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.304250 20940 tablet_replica.cc:440] T 0962a0a9b1f842b08b040f6e6a3c050e P a0258fb56e9d4b858dfa55326a152559: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0258fb56e9d4b858dfa55326a152559" member_type: VOTER last_known_addr { host: "127.19.180.1" port: 46735 } } peers { permanent_uuid: "1ab780ab6f274ded9a5fdac3c28998c7" member_type: VOTER last_known_addr { host: "127.19.180.2" port: 35451 } } peers { permanent_uuid: "c45c25ab0a664200896372dcc7565f59" member_type: VOTER last_known_addr { host: "127.19.180.3" port: 44881 } } }
15:02:53.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.304528 20940 tablet_replica.cc:443] T 0962a0a9b1f842b08b040f6e6a3c050e P a0258fb56e9d4b858dfa55326a152559: This TxnStatusTablet replica's current role is: FOLLOWER
15:02:53.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.392192 20939 raft_consensus.cc:491] T 0962a0a9b1f842b08b040f6e6a3c050e P c45c25ab0a664200896372dcc7565f59 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:02:53.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.392306 20940 raft_consensus.cc:491] T 0962a0a9b1f842b08b040f6e6a3c050e P a0258fb56e9d4b858dfa55326a152559 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:02:53.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.392442 20939 raft_consensus.cc:513] T 0962a0a9b1f842b08b040f6e6a3c050e P c45c25ab0a664200896372dcc7565f59 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0258fb56e9d4b858dfa55326a152559" member_type: VOTER last_known_addr { host: "127.19.180.1" port: 46735 } } peers { permanent_uuid: "1ab780ab6f274ded9a5fdac3c28998c7" member_type: VOTER last_known_addr { host: "127.19.180.2" port: 35451 } } peers { permanent_uuid: "c45c25ab0a664200896372dcc7565f59" member_type: VOTER last_known_addr { host: "127.19.180.3" port: 44881 } }
15:02:53.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.392468 20940 raft_consensus.cc:513] T 0962a0a9b1f842b08b040f6e6a3c050e P a0258fb56e9d4b858dfa55326a152559 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0258fb56e9d4b858dfa55326a152559" member_type: VOTER last_known_addr { host: "127.19.180.1" port: 46735 } } peers { permanent_uuid: "1ab780ab6f274ded9a5fdac3c28998c7" member_type: VOTER last_known_addr { host: "127.19.180.2" port: 35451 } } peers { permanent_uuid: "c45c25ab0a664200896372dcc7565f59" member_type: VOTER last_known_addr { host: "127.19.180.3" port: 44881 } }
15:02:53.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.393151 20939 leader_election.cc:290] T 0962a0a9b1f842b08b040f6e6a3c050e P c45c25ab0a664200896372dcc7565f59 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a0258fb56e9d4b858dfa55326a152559 (127.19.180.1:46735), 1ab780ab6f274ded9a5fdac3c28998c7 (127.19.180.2:35451)
15:02:53.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.393158 20940 leader_election.cc:290] T 0962a0a9b1f842b08b040f6e6a3c050e P a0258fb56e9d4b858dfa55326a152559 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1ab780ab6f274ded9a5fdac3c28998c7 (127.19.180.2:35451), c45c25ab0a664200896372dcc7565f59 (127.19.180.3:44881)
15:02:53.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.393684 20576 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0962a0a9b1f842b08b040f6e6a3c050e" candidate_uuid: "c45c25ab0a664200896372dcc7565f59" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a0258fb56e9d4b858dfa55326a152559" is_pre_election: true
15:02:53.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.393769 20718 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0962a0a9b1f842b08b040f6e6a3c050e" candidate_uuid: "a0258fb56e9d4b858dfa55326a152559" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1ab780ab6f274ded9a5fdac3c28998c7" is_pre_election: true
15:02:53.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.393908 20576 raft_consensus.cc:2463] T 0962a0a9b1f842b08b040f6e6a3c050e P a0258fb56e9d4b858dfa55326a152559 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c45c25ab0a664200896372dcc7565f59 in term 0.
15:02:53.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.393800 20717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0962a0a9b1f842b08b040f6e6a3c050e" candidate_uuid: "c45c25ab0a664200896372dcc7565f59" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1ab780ab6f274ded9a5fdac3c28998c7" is_pre_election: true
15:02:53.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.393973 20859 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0962a0a9b1f842b08b040f6e6a3c050e" candidate_uuid: "a0258fb56e9d4b858dfa55326a152559" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c45c25ab0a664200896372dcc7565f59" is_pre_election: true
15:02:53.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.394052 20718 raft_consensus.cc:2463] T 0962a0a9b1f842b08b040f6e6a3c050e P 1ab780ab6f274ded9a5fdac3c28998c7 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a0258fb56e9d4b858dfa55326a152559 in term 0.
15:02:53.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.394181 20859 raft_consensus.cc:2463] T 0962a0a9b1f842b08b040f6e6a3c050e P c45c25ab0a664200896372dcc7565f59 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a0258fb56e9d4b858dfa55326a152559 in term 0.
15:02:53.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.394524 20499 leader_election.cc:304] T 0962a0a9b1f842b08b040f6e6a3c050e P a0258fb56e9d4b858dfa55326a152559 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1ab780ab6f274ded9a5fdac3c28998c7, a0258fb56e9d4b858dfa55326a152559; no voters:
15:02:53.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.394830 20940 raft_consensus.cc:2798] T 0962a0a9b1f842b08b040f6e6a3c050e P a0258fb56e9d4b858dfa55326a152559 [term 0 FOLLOWER]: Leader pre-election won for term 1
15:02:53.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.394917 20940 raft_consensus.cc:491] T 0962a0a9b1f842b08b040f6e6a3c050e P a0258fb56e9d4b858dfa55326a152559 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
15:02:53.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.394976 20783 leader_election.cc:304] T 0962a0a9b1f842b08b040f6e6a3c050e P c45c25ab0a664200896372dcc7565f59 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: a0258fb56e9d4b858dfa55326a152559, c45c25ab0a664200896372dcc7565f59; no voters: 1ab780ab6f274ded9a5fdac3c28998c7
15:02:53.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.395037 20940 raft_consensus.cc:3054] T 0962a0a9b1f842b08b040f6e6a3c050e P a0258fb56e9d4b858dfa55326a152559 [term 0 FOLLOWER]: Advancing to term 1
15:02:53.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.395239 20939 raft_consensus.cc:2798] T 0962a0a9b1f842b08b040f6e6a3c050e P c45c25ab0a664200896372dcc7565f59 [term 0 FOLLOWER]: Leader pre-election won for term 1
15:02:53.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.395334 20939 raft_consensus.cc:491] T 0962a0a9b1f842b08b040f6e6a3c050e P c45c25ab0a664200896372dcc7565f59 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
15:02:53.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.395421 20939 raft_consensus.cc:3054] T 0962a0a9b1f842b08b040f6e6a3c050e P c45c25ab0a664200896372dcc7565f59 [term 0 FOLLOWER]: Advancing to term 1
15:02:53.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.396898 20940 raft_consensus.cc:513] T 0962a0a9b1f842b08b040f6e6a3c050e P a0258fb56e9d4b858dfa55326a152559 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0258fb56e9d4b858dfa55326a152559" member_type: VOTER last_known_addr { host: "127.19.180.1" port: 46735 } } peers { permanent_uuid: "1ab780ab6f274ded9a5fdac3c28998c7" member_type: VOTER last_known_addr { host: "127.19.180.2" port: 35451 } } peers { permanent_uuid: "c45c25ab0a664200896372dcc7565f59" member_type: VOTER last_known_addr { host: "127.19.180.3" port: 44881 } }
15:02:53.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.397097 20939 raft_consensus.cc:513] T 0962a0a9b1f842b08b040f6e6a3c050e P c45c25ab0a664200896372dcc7565f59 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0258fb56e9d4b858dfa55326a152559" member_type: VOTER last_known_addr { host: "127.19.180.1" port: 46735 } } peers { permanent_uuid: "1ab780ab6f274ded9a5fdac3c28998c7" member_type: VOTER last_known_addr { host: "127.19.180.2" port: 35451 } } peers { permanent_uuid: "c45c25ab0a664200896372dcc7565f59" member_type: VOTER last_known_addr { host: "127.19.180.3" port: 44881 } }
15:02:53.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.397469 20940 leader_election.cc:290] T 0962a0a9b1f842b08b040f6e6a3c050e P a0258fb56e9d4b858dfa55326a152559 [CANDIDATE]: Term 1 election: Requested vote from peers 1ab780ab6f274ded9a5fdac3c28998c7 (127.19.180.2:35451), c45c25ab0a664200896372dcc7565f59 (127.19.180.3:44881)
15:02:53.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.397675 20939 leader_election.cc:290] T 0962a0a9b1f842b08b040f6e6a3c050e P c45c25ab0a664200896372dcc7565f59 [CANDIDATE]: Term 1 election: Requested vote from peers a0258fb56e9d4b858dfa55326a152559 (127.19.180.1:46735), 1ab780ab6f274ded9a5fdac3c28998c7 (127.19.180.2:35451)
15:02:53.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.397895 20717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0962a0a9b1f842b08b040f6e6a3c050e" candidate_uuid: "a0258fb56e9d4b858dfa55326a152559" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1ab780ab6f274ded9a5fdac3c28998c7"
15:02:53.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.397946 20859 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0962a0a9b1f842b08b040f6e6a3c050e" candidate_uuid: "a0258fb56e9d4b858dfa55326a152559" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c45c25ab0a664200896372dcc7565f59"
15:02:53.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.398030 20576 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0962a0a9b1f842b08b040f6e6a3c050e" candidate_uuid: "c45c25ab0a664200896372dcc7565f59" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a0258fb56e9d4b858dfa55326a152559"
15:02:53.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.398102 20717 raft_consensus.cc:3054] T 0962a0a9b1f842b08b040f6e6a3c050e P 1ab780ab6f274ded9a5fdac3c28998c7 [term 0 FOLLOWER]: Advancing to term 1
15:02:53.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.398216 20859 raft_consensus.cc:2388] T 0962a0a9b1f842b08b040f6e6a3c050e P c45c25ab0a664200896372dcc7565f59 [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate a0258fb56e9d4b858dfa55326a152559 in current term 1: Already voted for candidate c45c25ab0a664200896372dcc7565f59 in this term.
15:02:53.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.398270 20576 raft_consensus.cc:2388] T 0962a0a9b1f842b08b040f6e6a3c050e P a0258fb56e9d4b858dfa55326a152559 [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate c45c25ab0a664200896372dcc7565f59 in current term 1: Already voted for candidate a0258fb56e9d4b858dfa55326a152559 in this term.
15:02:53.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.398836 20718 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0962a0a9b1f842b08b040f6e6a3c050e" candidate_uuid: "c45c25ab0a664200896372dcc7565f59" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1ab780ab6f274ded9a5fdac3c28998c7"
15:02:53.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.399348 20783 leader_election.cc:304] T 0962a0a9b1f842b08b040f6e6a3c050e P c45c25ab0a664200896372dcc7565f59 [CANDIDATE]: Term 1 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: c45c25ab0a664200896372dcc7565f59; no voters: 1ab780ab6f274ded9a5fdac3c28998c7, a0258fb56e9d4b858dfa55326a152559
15:02:53.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.399592 20939 raft_consensus.cc:2743] T 0962a0a9b1f842b08b040f6e6a3c050e P c45c25ab0a664200896372dcc7565f59 [term 1 FOLLOWER]: Leader election lost for term 1. Reason: could not achieve majority
15:02:53.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.400475 20717 raft_consensus.cc:2463] T 0962a0a9b1f842b08b040f6e6a3c050e P 1ab780ab6f274ded9a5fdac3c28998c7 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a0258fb56e9d4b858dfa55326a152559 in term 1.
15:02:53.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.400821 20499 leader_election.cc:304] T 0962a0a9b1f842b08b040f6e6a3c050e P a0258fb56e9d4b858dfa55326a152559 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 1ab780ab6f274ded9a5fdac3c28998c7, a0258fb56e9d4b858dfa55326a152559; no voters: c45c25ab0a664200896372dcc7565f59
15:02:53.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.401062 20940 raft_consensus.cc:2798] T 0962a0a9b1f842b08b040f6e6a3c050e P a0258fb56e9d4b858dfa55326a152559 [term 1 FOLLOWER]: Leader election won for term 1
15:02:53.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.401194 20940 raft_consensus.cc:695] T 0962a0a9b1f842b08b040f6e6a3c050e P a0258fb56e9d4b858dfa55326a152559 [term 1 LEADER]: Becoming Leader. State: Replica: a0258fb56e9d4b858dfa55326a152559, State: Running, Role: LEADER
15:02:53.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.401412 20940 consensus_queue.cc:237] T 0962a0a9b1f842b08b040f6e6a3c050e P a0258fb56e9d4b858dfa55326a152559 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0258fb56e9d4b858dfa55326a152559" member_type: VOTER last_known_addr { host: "127.19.180.1" port: 46735 } } peers { permanent_uuid: "1ab780ab6f274ded9a5fdac3c28998c7" member_type: VOTER last_known_addr { host: "127.19.180.2" port: 35451 } } peers { permanent_uuid: "c45c25ab0a664200896372dcc7565f59" member_type: VOTER last_known_addr { host: "127.19.180.3" port: 44881 } }
15:02:53.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.403014 20945 tablet_replica.cc:440] T 0962a0a9b1f842b08b040f6e6a3c050e P a0258fb56e9d4b858dfa55326a152559: TxnStatusTablet state changed. Reason: New leader a0258fb56e9d4b858dfa55326a152559. Latest consensus state: current_term: 1 leader_uuid: "a0258fb56e9d4b858dfa55326a152559" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0258fb56e9d4b858dfa55326a152559" member_type: VOTER last_known_addr { host: "127.19.180.1" port: 46735 } } peers { permanent_uuid: "1ab780ab6f274ded9a5fdac3c28998c7" member_type: VOTER last_known_addr { host: "127.19.180.2" port: 35451 } } peers { permanent_uuid: "c45c25ab0a664200896372dcc7565f59" member_type: VOTER last_known_addr { host: "127.19.180.3" port: 44881 } } }
15:02:53.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.403394 20945 tablet_replica.cc:443] T 0962a0a9b1f842b08b040f6e6a3c050e P a0258fb56e9d4b858dfa55326a152559: This TxnStatusTablet replica's current role is: LEADER
15:02:53.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.404363 20961 txn_status_manager.cc:874] Waiting until node catch up with all replicated operations in previous term...
15:02:53.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.404708 20961 txn_status_manager.cc:930] Loading transaction status metadata into memory...
15:02:53.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.405710 20297 catalog_manager.cc:5526] T 0962a0a9b1f842b08b040f6e6a3c050e P a0258fb56e9d4b858dfa55326a152559 reported cstate change: term changed from 0 to 1, leader changed from <none> to a0258fb56e9d4b858dfa55326a152559 (127.19.180.1). New cstate: current_term: 1 leader_uuid: "a0258fb56e9d4b858dfa55326a152559" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0258fb56e9d4b858dfa55326a152559" member_type: VOTER last_known_addr { host: "127.19.180.1" port: 46735 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1ab780ab6f274ded9a5fdac3c28998c7" member_type: VOTER last_known_addr { host: "127.19.180.2" port: 35451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c45c25ab0a664200896372dcc7565f59" member_type: VOTER last_known_addr { host: "127.19.180.3" port: 44881 } health_report { overall_health: UNKNOWN } } }
15:02:53.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.438886 20945 consensus_queue.cc:1035] T fb15a93555ef4c018b3028f224e8e83b P a0258fb56e9d4b858dfa55326a152559 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c45c25ab0a664200896372dcc7565f59" member_type: VOTER last_known_addr { host: "127.19.180.3" port: 44881 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:02:53.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.457479 20717 raft_consensus.cc:1270] T 0962a0a9b1f842b08b040f6e6a3c050e P 1ab780ab6f274ded9a5fdac3c28998c7 [term 1 FOLLOWER]: Refusing update from remote peer a0258fb56e9d4b858dfa55326a152559: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
15:02:53.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.458384 20945 consensus_queue.cc:1035] T fb15a93555ef4c018b3028f224e8e83b P a0258fb56e9d4b858dfa55326a152559 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ab780ab6f274ded9a5fdac3c28998c7" member_type: VOTER last_known_addr { host: "127.19.180.2" port: 35451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:02:53.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.458803 20859 raft_consensus.cc:1270] T 0962a0a9b1f842b08b040f6e6a3c050e P c45c25ab0a664200896372dcc7565f59 [term 1 FOLLOWER]: Refusing update from remote peer a0258fb56e9d4b858dfa55326a152559: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
15:02:53.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.458976 20945 consensus_queue.cc:1035] T 0962a0a9b1f842b08b040f6e6a3c050e P a0258fb56e9d4b858dfa55326a152559 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ab780ab6f274ded9a5fdac3c28998c7" member_type: VOTER last_known_addr { host: "127.19.180.2" port: 35451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:02:53.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.462002 20940 consensus_queue.cc:1035] T 0962a0a9b1f842b08b040f6e6a3c050e P a0258fb56e9d4b858dfa55326a152559 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c45c25ab0a664200896372dcc7565f59" member_type: VOTER last_known_addr { host: "127.19.180.3" port: 44881 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:02:53.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.483064 20939 tablet_replica.cc:440] T 0962a0a9b1f842b08b040f6e6a3c050e P c45c25ab0a664200896372dcc7565f59: TxnStatusTablet state changed. Reason: New leader a0258fb56e9d4b858dfa55326a152559. Latest consensus state: current_term: 1 leader_uuid: "a0258fb56e9d4b858dfa55326a152559" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0258fb56e9d4b858dfa55326a152559" member_type: VOTER last_known_addr { host: "127.19.180.1" port: 46735 } } peers { permanent_uuid: "1ab780ab6f274ded9a5fdac3c28998c7" member_type: VOTER last_known_addr { host: "127.19.180.2" port: 35451 } } peers { permanent_uuid: "c45c25ab0a664200896372dcc7565f59" member_type: VOTER last_known_addr { host: "127.19.180.3" port: 44881 } } }
15:02:53.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.483405 20939 tablet_replica.cc:443] T 0962a0a9b1f842b08b040f6e6a3c050e P c45c25ab0a664200896372dcc7565f59: This TxnStatusTablet replica's current role is: FOLLOWER
15:02:53.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.489797 20945 tablet_replica.cc:440] T 0962a0a9b1f842b08b040f6e6a3c050e P a0258fb56e9d4b858dfa55326a152559: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "a0258fb56e9d4b858dfa55326a152559" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0258fb56e9d4b858dfa55326a152559" member_type: VOTER last_known_addr { host: "127.19.180.1" port: 46735 } } peers { permanent_uuid: "1ab780ab6f274ded9a5fdac3c28998c7" member_type: VOTER last_known_addr { host: "127.19.180.2" port: 35451 } } peers { permanent_uuid: "c45c25ab0a664200896372dcc7565f59" member_type: VOTER last_known_addr { host: "127.19.180.3" port: 44881 } } }
15:02:53.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.490056 20945 tablet_replica.cc:443] T 0962a0a9b1f842b08b040f6e6a3c050e P a0258fb56e9d4b858dfa55326a152559: This TxnStatusTablet replica's current role is: LEADER
15:02:53.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.490172 20966 mvcc.cc:204] Tried to move back new op lower bound from 7113499334464020480 to 7113499334254460928. Current Snapshot: MvccSnapshot[applied={T|T < 7113499334464020480}]
15:02:53.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.492204 20939 tablet_replica.cc:440] T 0962a0a9b1f842b08b040f6e6a3c050e P c45c25ab0a664200896372dcc7565f59: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "a0258fb56e9d4b858dfa55326a152559" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0258fb56e9d4b858dfa55326a152559" member_type: VOTER last_known_addr { host: "127.19.180.1" port: 46735 } } peers { permanent_uuid: "1ab780ab6f274ded9a5fdac3c28998c7" member_type: VOTER last_known_addr { host: "127.19.180.2" port: 35451 } } peers { permanent_uuid: "c45c25ab0a664200896372dcc7565f59" member_type: VOTER last_known_addr { host: "127.19.180.3" port: 44881 } } }
15:02:53.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.492436 20939 tablet_replica.cc:443] T 0962a0a9b1f842b08b040f6e6a3c050e P c45c25ab0a664200896372dcc7565f59: This TxnStatusTablet replica's current role is: FOLLOWER
15:02:53.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.499457 20937 tablet_replica.cc:440] T 0962a0a9b1f842b08b040f6e6a3c050e P 1ab780ab6f274ded9a5fdac3c28998c7: TxnStatusTablet state changed. Reason: New leader a0258fb56e9d4b858dfa55326a152559. Latest consensus state: current_term: 1 leader_uuid: "a0258fb56e9d4b858dfa55326a152559" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0258fb56e9d4b858dfa55326a152559" member_type: VOTER last_known_addr { host: "127.19.180.1" port: 46735 } } peers { permanent_uuid: "1ab780ab6f274ded9a5fdac3c28998c7" member_type: VOTER last_known_addr { host: "127.19.180.2" port: 35451 } } peers { permanent_uuid: "c45c25ab0a664200896372dcc7565f59" member_type: VOTER last_known_addr { host: "127.19.180.3" port: 44881 } } }
15:02:53.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.499761 20937 tablet_replica.cc:443] T 0962a0a9b1f842b08b040f6e6a3c050e P 1ab780ab6f274ded9a5fdac3c28998c7: This TxnStatusTablet replica's current role is: FOLLOWER
15:02:53.502 [DEBUG - Time-limited test] (KuduTransaction.java:702) starting keepalive heartbeating with period 500 ms (txn ID 0)
15:02:53.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.500785 20945 tablet_replica.cc:440] T 0962a0a9b1f842b08b040f6e6a3c050e P a0258fb56e9d4b858dfa55326a152559: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "a0258fb56e9d4b858dfa55326a152559" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0258fb56e9d4b858dfa55326a152559" member_type: VOTER last_known_addr { host: "127.19.180.1" port: 46735 } } peers { permanent_uuid: "1ab780ab6f274ded9a5fdac3c28998c7" member_type: VOTER last_known_addr { host: "127.19.180.2" port: 35451 } } peers { permanent_uuid: "c45c25ab0a664200896372dcc7565f59" member_type: VOTER last_known_addr { host: "127.19.180.3" port: 44881 } } }
15:02:53.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.501040 20945 tablet_replica.cc:443] T 0962a0a9b1f842b08b040f6e6a3c050e P a0258fb56e9d4b858dfa55326a152559: This TxnStatusTablet replica's current role is: LEADER
15:02:53.505 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table 221d15de79984a85af5c16c6e92777a4
15:02:53.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.506618 20937 tablet_replica.cc:440] T 0962a0a9b1f842b08b040f6e6a3c050e P 1ab780ab6f274ded9a5fdac3c28998c7: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "a0258fb56e9d4b858dfa55326a152559" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0258fb56e9d4b858dfa55326a152559" member_type: VOTER last_known_addr { host: "127.19.180.1" port: 46735 } } peers { permanent_uuid: "1ab780ab6f274ded9a5fdac3c28998c7" member_type: VOTER last_known_addr { host: "127.19.180.2" port: 35451 } } peers { permanent_uuid: "c45c25ab0a664200896372dcc7565f59" member_type: VOTER last_known_addr { host: "127.19.180.3" port: 44881 } } }
15:02:53.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.506815 20937 tablet_replica.cc:443] T 0962a0a9b1f842b08b040f6e6a3c050e P 1ab780ab6f274ded9a5fdac3c28998c7: This TxnStatusTablet replica's current role is: FOLLOWER
15:02:53.510 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet fb15a93555ef4c018b3028f224e8e83b for table 'txn_manager_fallback' with partition [0x00000000, 0x00000001)
15:02:53.512 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 9727c0ccc99e478eb0c34096f44de1a6 for table 'txn_manager_fallback' with partition [0x00000001, 0x00000002)
15:02:53.512 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x00000000, upperBoundPartitionKey=0x00000001, ttl=299999, tablet=fb15a93555ef4c018b3028f224e8e83b@[1ab780ab6f274ded9a5fdac3c28998c7(127.19.180.2:35451),a0258fb56e9d4b858dfa55326a152559(127.19.180.1:46735)[L],c45c25ab0a664200896372dcc7565f59(127.19.180.3:44881)]}, Tablet{lowerBoundPartitionKey=0x00000001, upperBoundPartitionKey=0x00000002, ttl=299999, tablet=9727c0ccc99e478eb0c34096f44de1a6@[1ab780ab6f274ded9a5fdac3c28998c7(127.19.180.2:35451),a0258fb56e9d4b858dfa55326a152559(127.19.180.1:46735),c45c25ab0a664200896372dcc7565f59(127.19.180.3:44881)[L]]}, NonCoveredRange{lowerBoundPartitionKey=0x00000002, upperBoundPartitionKey=0x, ttl=299999}]
15:02:53.512 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=6), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC GetTableLocations: sending RPC to server master-127.19.180.61:37725, [5ms] Sub RPC GetTableLocations: received response from server master-127.19.180.61:37725: OK, deferred=null) after lookup
15:02:53.513 [DEBUG - kudu-nio-1] (Connection.java:748) [peer a0258fb56e9d4b858dfa55326a152559(127.19.180.1:46735)] connecting to peer
15:02:53.514 [DEBUG - kudu-nio-1] (Connection.java:757) [peer a0258fb56e9d4b858dfa55326a152559(127.19.180.1:46735)] Successfully connected to peer
15:02:53.526 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x797a6ec2, L:/127.0.0.1:44490 - R:/127.19.180.1:46735] using TOKEN/null
15:02:53.574 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.19.180.61:37725
15:02:53.575 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
15:02:53.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.576489 20176 external_mini_cluster.cc:1661] Attempting to check leaks for /tmp/dist-test-taskDlDeJ5/build/asan/bin/kudu pid 20265
15:02:53.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.577782 20339 generic_service.cc:196] Checking for leaks (request via RPC)
15:02:53.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.599346 20939 consensus_queue.cc:1035] T 9727c0ccc99e478eb0c34096f44de1a6 P c45c25ab0a664200896372dcc7565f59 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ab780ab6f274ded9a5fdac3c28998c7" member_type: VOTER last_known_addr { host: "127.19.180.2" port: 35451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:02:53.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.611446 20939 consensus_queue.cc:1035] T 9727c0ccc99e478eb0c34096f44de1a6 P c45c25ab0a664200896372dcc7565f59 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a0258fb56e9d4b858dfa55326a152559" member_type: VOTER last_known_addr { host: "127.19.180.1" port: 46735 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:02:53.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:53.699520 20176 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskDlDeJ5/build/asan/bin/kudu with pid 20265
15:02:53.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:53.717242 20906 heartbeater.cc:643] Failed to heartbeat to 127.19.180.61:37725 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv got EOF from 127.19.180.61:37725 (error 108)
15:02:53.717 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.19.180.61:37725(127.19.180.61:37725)] handling channelInactive
15:02:53.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:53.717589 20765 heartbeater.cc:643] Failed to heartbeat to 127.19.180.61:37725 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv got EOF from 127.19.180.61:37725 (error 108)
15:02:53.717 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.180.61:37725(127.19.180.61:37725)] cleaning up while in state READY due to: connection closed
15:02:53.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:53.718725 20624 heartbeater.cc:643] Failed to heartbeat to 127.19.180.61:37725 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv got EOF from 127.19.180.61:37725 (error 108)
15:02:53.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:53.719300 20367 connection.cc:537] server connection from 127.0.0.1:40732 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
15:02:53.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:53.719705 20205 connection.cc:537] server connection from 127.0.0.1:51364 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
15:02:53.720 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:02:53.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:53.720999 20984 negotiation.cc:337] Failed RPC negotiation. Trace:
15:02:53.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:02:53.718185 (+ 0us) reactor.cc:625] Submitting negotiation task for client connection to 127.19.180.61:37725 (local address 127.19.180.2:59697)
15:02:53.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:02:53.718708 (+ 523us) negotiation.cc:107] Waiting for socket to connect
15:02:53.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:02:53.718726 (+ 18us) client_negotiation.cc:174] Beginning negotiation
15:02:53.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:02:53.718835 (+ 109us) client_negotiation.cc:252] Sending NEGOTIATE NegotiatePB request
15:02:53.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:02:53.720109 (+ 1274us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.19.180.61:37725: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
15:02:53.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"client-negotiator.queue_time_us":411,"thread_start_us":284,"threads_started":1}
15:02:53.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:53.722220 20985 negotiation.cc:337] Failed RPC negotiation. Trace:
15:02:53.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:02:53.719343 (+ 0us) reactor.cc:625] Submitting negotiation task for client connection to 127.19.180.61:37725 (local address 127.19.180.1:52947)
15:02:53.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:02:53.721020 (+ 1677us) negotiation.cc:107] Waiting for socket to connect
15:02:53.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:02:53.721232 (+ 212us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.19.180.61:37725: connect: Connection reset by peer (error 104)
15:02:53.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"client-negotiator.queue_time_us":1556,"thread_start_us":1406,"threads_started":1}
15:02:54.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.000334 20986 raft_consensus.cc:491] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader ff11b88709b14cda87f5cecd80a792d6)
15:02:54.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.000560 20986 raft_consensus.cc:513] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3ea36a1a750c40b98e9e225442fadbd1" member_type: VOTER last_known_addr { host: "127.19.180.62" port: 46081 } } peers { permanent_uuid: "ff11b88709b14cda87f5cecd80a792d6" member_type: VOTER last_known_addr { host: "127.19.180.61" port: 37725 } } peers { permanent_uuid: "528e2c60e3bc4a5aa2ba21cac324d896" member_type: VOTER last_known_addr { host: "127.19.180.60" port: 32797 } }
15:02:54.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.001828 20987 raft_consensus.cc:491] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader ff11b88709b14cda87f5cecd80a792d6)
15:02:54.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.001901 20986 leader_election.cc:290] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers ff11b88709b14cda87f5cecd80a792d6 (127.19.180.61:37725), 528e2c60e3bc4a5aa2ba21cac324d896 (127.19.180.60:32797)
15:02:54.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.002233 20987 raft_consensus.cc:513] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3ea36a1a750c40b98e9e225442fadbd1" member_type: VOTER last_known_addr { host: "127.19.180.62" port: 46081 } } peers { permanent_uuid: "ff11b88709b14cda87f5cecd80a792d6" member_type: VOTER last_known_addr { host: "127.19.180.61" port: 37725 } } peers { permanent_uuid: "528e2c60e3bc4a5aa2ba21cac324d896" member_type: VOTER last_known_addr { host: "127.19.180.60" port: 32797 } }
15:02:54.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.002589 20390 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3ea36a1a750c40b98e9e225442fadbd1" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "528e2c60e3bc4a5aa2ba21cac324d896" is_pre_election: true
15:02:54.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.002898 20390 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3ea36a1a750c40b98e9e225442fadbd1 in term 1.
15:02:54.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.003680 20987 leader_election.cc:290] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 3ea36a1a750c40b98e9e225442fadbd1 (127.19.180.62:46081), ff11b88709b14cda87f5cecd80a792d6 (127.19.180.61:37725)
15:02:54.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.003535 20205 leader_election.cc:304] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3ea36a1a750c40b98e9e225442fadbd1, 528e2c60e3bc4a5aa2ba21cac324d896; no voters:
15:02:54.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.004079 20228 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "528e2c60e3bc4a5aa2ba21cac324d896" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "3ea36a1a750c40b98e9e225442fadbd1" is_pre_election: true
15:02:54.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.004086 20986 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1 [term 1 FOLLOWER]: Leader pre-election won for term 2
15:02:54.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.004457 20228 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 528e2c60e3bc4a5aa2ba21cac324d896 in term 1.
15:02:54.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:54.004525 20203 leader_election.cc:336] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer ff11b88709b14cda87f5cecd80a792d6 (127.19.180.61:37725): Network error: Client connection negotiation failed: client connection to 127.19.180.61:37725: connect: Connection refused (error 111)
15:02:54.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.004791 20986 raft_consensus.cc:491] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1 [term 1 FOLLOWER]: Starting leader election (detected failure of leader ff11b88709b14cda87f5cecd80a792d6)
15:02:54.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:54.004755 20365 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.180.61:37725: connect: Connection refused (error 111)
15:02:54.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.004899 20986 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1 [term 1 FOLLOWER]: Advancing to term 2
15:02:54.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.005012 20367 leader_election.cc:304] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3ea36a1a750c40b98e9e225442fadbd1, 528e2c60e3bc4a5aa2ba21cac324d896; no voters:
15:02:54.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.005519 20987 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [term 1 FOLLOWER]: Leader pre-election won for term 2
15:02:54.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.005638 20987 raft_consensus.cc:491] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [term 1 FOLLOWER]: Starting leader election (detected failure of leader ff11b88709b14cda87f5cecd80a792d6)
15:02:54.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.005745 20987 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [term 1 FOLLOWER]: Advancing to term 2
15:02:54.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:54.006165 20365 leader_election.cc:336] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer ff11b88709b14cda87f5cecd80a792d6 (127.19.180.61:37725): Network error: Client connection negotiation failed: client connection to 127.19.180.61:37725: connect: Connection refused (error 111)
15:02:54.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.007342 20986 raft_consensus.cc:513] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3ea36a1a750c40b98e9e225442fadbd1" member_type: VOTER last_known_addr { host: "127.19.180.62" port: 46081 } } peers { permanent_uuid: "ff11b88709b14cda87f5cecd80a792d6" member_type: VOTER last_known_addr { host: "127.19.180.61" port: 37725 } } peers { permanent_uuid: "528e2c60e3bc4a5aa2ba21cac324d896" member_type: VOTER last_known_addr { host: "127.19.180.60" port: 32797 } }
15:02:54.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.008077 20987 raft_consensus.cc:513] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3ea36a1a750c40b98e9e225442fadbd1" member_type: VOTER last_known_addr { host: "127.19.180.62" port: 46081 } } peers { permanent_uuid: "ff11b88709b14cda87f5cecd80a792d6" member_type: VOTER last_known_addr { host: "127.19.180.61" port: 37725 } } peers { permanent_uuid: "528e2c60e3bc4a5aa2ba21cac324d896" member_type: VOTER last_known_addr { host: "127.19.180.60" port: 32797 } }
15:02:54.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.008611 20986 leader_election.cc:290] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1 [CANDIDATE]: Term 2 election: Requested vote from peers ff11b88709b14cda87f5cecd80a792d6 (127.19.180.61:37725), 528e2c60e3bc4a5aa2ba21cac324d896 (127.19.180.60:32797)
15:02:54.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.008723 20987 leader_election.cc:290] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [CANDIDATE]: Term 2 election: Requested vote from peers 3ea36a1a750c40b98e9e225442fadbd1 (127.19.180.62:46081), ff11b88709b14cda87f5cecd80a792d6 (127.19.180.61:37725)
15:02:54.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.009121 20390 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3ea36a1a750c40b98e9e225442fadbd1" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "528e2c60e3bc4a5aa2ba21cac324d896"
15:02:54.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.009286 20228 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "528e2c60e3bc4a5aa2ba21cac324d896" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "3ea36a1a750c40b98e9e225442fadbd1"
15:02:54.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.009385 20390 raft_consensus.cc:2388] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 3ea36a1a750c40b98e9e225442fadbd1 in current term 2: Already voted for candidate 528e2c60e3bc4a5aa2ba21cac324d896 in this term.
15:02:54.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.009510 20228 raft_consensus.cc:2388] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 528e2c60e3bc4a5aa2ba21cac324d896 in current term 2: Already voted for candidate 3ea36a1a750c40b98e9e225442fadbd1 in this term.
15:02:54.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:54.010131 20203 leader_election.cc:336] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer ff11b88709b14cda87f5cecd80a792d6 (127.19.180.61:37725): Network error: Client connection negotiation failed: client connection to 127.19.180.61:37725: connect: Connection refused (error 111)
15:02:54.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.010251 20203 leader_election.cc:304] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3ea36a1a750c40b98e9e225442fadbd1; no voters: 528e2c60e3bc4a5aa2ba21cac324d896, ff11b88709b14cda87f5cecd80a792d6
15:02:54.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.010829 20986 raft_consensus.cc:2743] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
15:02:54.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:54.011329 20365 leader_election.cc:336] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer ff11b88709b14cda87f5cecd80a792d6 (127.19.180.61:37725): Network error: Client connection negotiation failed: client connection to 127.19.180.61:37725: connect: Connection refused (error 111)
15:02:54.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.011457 20365 leader_election.cc:304] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 528e2c60e3bc4a5aa2ba21cac324d896; no voters: 3ea36a1a750c40b98e9e225442fadbd1, ff11b88709b14cda87f5cecd80a792d6
15:02:54.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.011766 20987 raft_consensus.cc:2743] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
15:02:54.012 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.61:37725(127.19.180.61:37725)] connecting to peer
15:02:54.013 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:54.013 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.180.61:37725(127.19.180.61:37725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:54.013 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location master-127.19.180.61:37725(127.19.180.61:37725) for tablet Kudu Master: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:54.028 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.61:37725(127.19.180.61:37725)] connecting to peer
15:02:54.029 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:54.029 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.180.61:37725(127.19.180.61:37725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:54.029 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.61:37725: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:54.029 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797; will retry
15:02:54.047 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.61:37725(127.19.180.61:37725)] connecting to peer
15:02:54.048 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:54.048 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.180.61:37725(127.19.180.61:37725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:54.049 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.61:37725: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:54.050 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797; will retry
15:02:54.067 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.61:37725(127.19.180.61:37725)] connecting to peer
15:02:54.068 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:54.068 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.180.61:37725(127.19.180.61:37725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:54.068 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.61:37725: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:54.069 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797; will retry
15:02:54.087 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.61:37725(127.19.180.61:37725)] connecting to peer
15:02:54.089 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:54.089 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.180.61:37725(127.19.180.61:37725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:54.089 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.61:37725: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:54.090 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797; will retry
15:02:54.127 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.61:37725(127.19.180.61:37725)] connecting to peer
15:02:54.129 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:54.129 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.180.61:37725(127.19.180.61:37725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:54.129 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.61:37725: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:54.131 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797; will retry
15:02:54.187 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.61:37725(127.19.180.61:37725)] connecting to peer
15:02:54.188 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:54.188 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.180.61:37725(127.19.180.61:37725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:54.188 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.61:37725: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:54.189 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797; will retry
15:02:54.268 [DEBUG - pool-38-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=7, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] sending RPC to server master-127.19.180.61:37725, [5ms] received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [5ms] delaying RPC due to: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: OK, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: OK, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: OK, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: OK, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: OK, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [123ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [180ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: OK, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, deferred=Deferred@845610087(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$83/2068800939@27e1fbd -> (continuation of Deferred@146101632 after retry RPC after error@875354113) -> (continuation of Deferred@825003815 after retry RPC after error@1130233160) -> (continuation of Deferred@1642506742 after retry RPC after error@869832987) -> (continuation of Deferred@1725459952 after retry RPC after error@1245936606) -> (continuation of Deferred@515822201 after retry RPC after error@1775866100) -> (continuation of Deferred@1280038866 after retry RPC after error@487053326), errback=org.apache.kudu.client.KuduTransaction$$Lambda$84/356594043@7542eccd -> (continuation of Deferred@146101632 after retry RPC after error@875354113) -> (continuation of Deferred@825003815 after retry RPC after error@1130233160) -> (continuation of Deferred@1642506742 after retry RPC after error@869832987) -> (continuation of Deferred@1725459952 after retry RPC after error@1245936606) -> (continuation of Deferred@515822201 after retry RPC after error@1775866100) -> (continuation of Deferred@1280038866 after retry RPC after error@487053326)))
15:02:54.269 [WARN - pool-38-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
15:02:54.288 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.61:37725(127.19.180.61:37725)] connecting to peer
15:02:54.289 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:54.289 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.180.61:37725(127.19.180.61:37725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:54.289 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.61:37725: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:54.290 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797; will retry
15:02:54.308 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.61:37725(127.19.180.61:37725)] connecting to peer
15:02:54.309 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:54.309 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.180.61:37725(127.19.180.61:37725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:54.309 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.61:37725: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:54.310 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797; will retry
15:02:54.328 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.61:37725(127.19.180.61:37725)] connecting to peer
15:02:54.328 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:54.329 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.180.61:37725(127.19.180.61:37725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:54.329 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.61:37725: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:54.330 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797; will retry
15:02:54.348 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.61:37725(127.19.180.61:37725)] connecting to peer
15:02:54.349 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:54.349 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.180.61:37725(127.19.180.61:37725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:54.350 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.61:37725: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:54.350 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797; will retry
15:02:54.367 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.61:37725(127.19.180.61:37725)] connecting to peer
15:02:54.368 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:54.369 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.180.61:37725(127.19.180.61:37725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:54.369 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.61:37725: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:54.369 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797; will retry
15:02:54.408 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.61:37725(127.19.180.61:37725)] connecting to peer
15:02:54.409 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:54.410 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.180.61:37725(127.19.180.61:37725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:54.410 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.61:37725: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:54.410 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797; will retry
15:02:54.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.424563 20986 raft_consensus.cc:491] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:02:54.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.424793 20986 raft_consensus.cc:513] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3ea36a1a750c40b98e9e225442fadbd1" member_type: VOTER last_known_addr { host: "127.19.180.62" port: 46081 } } peers { permanent_uuid: "ff11b88709b14cda87f5cecd80a792d6" member_type: VOTER last_known_addr { host: "127.19.180.61" port: 37725 } } peers { permanent_uuid: "528e2c60e3bc4a5aa2ba21cac324d896" member_type: VOTER last_known_addr { host: "127.19.180.60" port: 32797 } }
15:02:54.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.425515 20986 leader_election.cc:290] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers ff11b88709b14cda87f5cecd80a792d6 (127.19.180.61:37725), 528e2c60e3bc4a5aa2ba21cac324d896 (127.19.180.60:32797)
15:02:54.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.426151 20390 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3ea36a1a750c40b98e9e225442fadbd1" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "528e2c60e3bc4a5aa2ba21cac324d896" is_pre_election: true
15:02:54.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.426400 20390 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3ea36a1a750c40b98e9e225442fadbd1 in term 2.
15:02:54.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.426805 20205 leader_election.cc:304] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3ea36a1a750c40b98e9e225442fadbd1, 528e2c60e3bc4a5aa2ba21cac324d896; no voters:
15:02:54.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:54.427042 20203 leader_election.cc:336] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer ff11b88709b14cda87f5cecd80a792d6 (127.19.180.61:37725): Network error: Client connection negotiation failed: client connection to 127.19.180.61:37725: connect: Connection refused (error 111)
15:02:54.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.427170 20986 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1 [term 2 FOLLOWER]: Leader pre-election won for term 3
15:02:54.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.427271 20986 raft_consensus.cc:491] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
15:02:54.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.427383 20986 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1 [term 2 FOLLOWER]: Advancing to term 3
15:02:54.428 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.61:37725(127.19.180.61:37725)] connecting to peer
15:02:54.429 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:54.429 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.180.61:37725(127.19.180.61:37725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:54.429 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.61:37725: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:54.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.429677 20986 raft_consensus.cc:513] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3ea36a1a750c40b98e9e225442fadbd1" member_type: VOTER last_known_addr { host: "127.19.180.62" port: 46081 } } peers { permanent_uuid: "ff11b88709b14cda87f5cecd80a792d6" member_type: VOTER last_known_addr { host: "127.19.180.61" port: 37725 } } peers { permanent_uuid: "528e2c60e3bc4a5aa2ba21cac324d896" member_type: VOTER last_known_addr { host: "127.19.180.60" port: 32797 } }
15:02:54.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.430696 20390 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3ea36a1a750c40b98e9e225442fadbd1" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "528e2c60e3bc4a5aa2ba21cac324d896"
15:02:54.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.430902 20390 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [term 2 FOLLOWER]: Advancing to term 3
15:02:54.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.431653 20986 leader_election.cc:290] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1 [CANDIDATE]: Term 3 election: Requested vote from peers ff11b88709b14cda87f5cecd80a792d6 (127.19.180.61:37725), 528e2c60e3bc4a5aa2ba21cac324d896 (127.19.180.60:32797)
15:02:54.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:54.432154 20203 leader_election.cc:336] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer ff11b88709b14cda87f5cecd80a792d6 (127.19.180.61:37725): Network error: Client connection negotiation failed: client connection to 127.19.180.61:37725: connect: Connection refused (error 111)
15:02:54.432 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797; will retry
15:02:54.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.433312 20390 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3ea36a1a750c40b98e9e225442fadbd1 in term 3.
15:02:54.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.433693 20205 leader_election.cc:304] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3ea36a1a750c40b98e9e225442fadbd1, 528e2c60e3bc4a5aa2ba21cac324d896; no voters: ff11b88709b14cda87f5cecd80a792d6
15:02:54.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.433950 20986 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1 [term 3 FOLLOWER]: Leader election won for term 3
15:02:54.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.434370 20986 raft_consensus.cc:695] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1 [term 3 LEADER]: Becoming Leader. State: Replica: 3ea36a1a750c40b98e9e225442fadbd1, State: Running, Role: LEADER
15:02:54.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.434774 20986 consensus_queue.cc:237] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3ea36a1a750c40b98e9e225442fadbd1" member_type: VOTER last_known_addr { host: "127.19.180.62" port: 46081 } } peers { permanent_uuid: "ff11b88709b14cda87f5cecd80a792d6" member_type: VOTER last_known_addr { host: "127.19.180.61" port: 37725 } } peers { permanent_uuid: "528e2c60e3bc4a5aa2ba21cac324d896" member_type: VOTER last_known_addr { host: "127.19.180.60" port: 32797 } }
15:02:54.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.436365 20995 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 3ea36a1a750c40b98e9e225442fadbd1. Latest consensus state: current_term: 3 leader_uuid: "3ea36a1a750c40b98e9e225442fadbd1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3ea36a1a750c40b98e9e225442fadbd1" member_type: VOTER last_known_addr { host: "127.19.180.62" port: 46081 } } peers { permanent_uuid: "ff11b88709b14cda87f5cecd80a792d6" member_type: VOTER last_known_addr { host: "127.19.180.61" port: 37725 } } peers { permanent_uuid: "528e2c60e3bc4a5aa2ba21cac324d896" member_type: VOTER last_known_addr { host: "127.19.180.60" port: 32797 } } }
15:02:54.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.436725 20995 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1 [sys.catalog]: This master's current role is: LEADER
15:02:54.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.437330 20997 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
15:02:54.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.439635 20997 catalog_manager.cc:670] Loaded metadata for table txn_manager_fallback [id=221d15de79984a85af5c16c6e92777a4]
15:02:54.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.440099 20997 catalog_manager.cc:670] Loaded metadata for table kudu_system.kudu_transactions [id=48b774e56c0a48579a937d7b974ec75b]
15:02:54.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.441635 20997 tablet_loader.cc:96] loaded metadata for tablet 0962a0a9b1f842b08b040f6e6a3c050e (table kudu_system.kudu_transactions [id=48b774e56c0a48579a937d7b974ec75b])
15:02:54.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.442094 20997 tablet_loader.cc:96] loaded metadata for tablet 9727c0ccc99e478eb0c34096f44de1a6 (table txn_manager_fallback [id=221d15de79984a85af5c16c6e92777a4])
15:02:54.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.442466 20997 tablet_loader.cc:96] loaded metadata for tablet fb15a93555ef4c018b3028f224e8e83b (table txn_manager_fallback [id=221d15de79984a85af5c16c6e92777a4])
15:02:54.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.442718 20997 catalog_manager.cc:1485] Initializing Kudu cluster ID...
15:02:54.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.443778 20997 catalog_manager.cc:1260] Loaded cluster ID: e212c04cc2fe4a68b45af340d61c59d7
15:02:54.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.443871 20997 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
15:02:54.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.444965 20997 catalog_manager.cc:1505] Loading token signing keys...
15:02:54.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.445789 20997 catalog_manager.cc:5910] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1: Loaded TSK: 0
15:02:54.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.446596 20997 catalog_manager.cc:1515] Initializing in-progress tserver states...
15:02:54.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:54.517607 20203 consensus_peers.cc:487] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1 -> Peer ff11b88709b14cda87f5cecd80a792d6 (127.19.180.61:37725): Couldn't send request to peer ff11b88709b14cda87f5cecd80a792d6. Status: Network error: Client connection negotiation failed: client connection to 127.19.180.61:37725: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
15:02:54.527 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.61:37725(127.19.180.61:37725)] connecting to peer
15:02:54.528 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:54.528 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.180.61:37725(127.19.180.61:37725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:54.528 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.61:37725: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:54.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.528890 20390 raft_consensus.cc:1270] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [term 3 FOLLOWER]: Refusing update from remote peer 3ea36a1a750c40b98e9e225442fadbd1: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 3 index: 12. (index mismatch)
15:02:54.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.529718 20995 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "528e2c60e3bc4a5aa2ba21cac324d896" member_type: VOTER last_known_addr { host: "127.19.180.60" port: 32797 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
15:02:54.530 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
15:02:54.531 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: OK, [3ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: OK, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: OK, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: OK, [123ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [123ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [143ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [145ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: OK, [145ms] delaying RPC due to: Service unavailable: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [243ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: OK, deferred=Deferred@60665556(state=PENDING, result=null, callback=(continuation of Deferred@1129259573 after retry RPC after error@1589893570) -> (continuation of Deferred@486665818 after retry RPC after error@1938032672) -> (continuation of Deferred@1231082223 after retry RPC after error@162336712) -> (continuation of Deferred@466371481 after retry RPC after error@1213894088) -> (continuation of Deferred@123918726 after retry RPC after error@1724929517) -> (continuation of Deferred@528028197 after retry RPC after error@64092367) -> (continuation of Deferred@317636570 after retry RPC after error@1995829079), errback=(continuation of Deferred@1129259573 after retry RPC after error@1589893570) -> (continuation of Deferred@486665818 after retry RPC after error@1938032672) -> (continuation of Deferred@1231082223 after retry RPC after error@162336712) -> (continuation of Deferred@466371481 after retry RPC after error@1213894088) -> (continuation of Deferred@123918726 after retry RPC after error@1724929517) -> (continuation of Deferred@528028197 after retry RPC after error@64092367) -> (continuation of Deferred@317636570 after retry RPC after error@1995829079))) after lookup
15:02:54.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.531760 20998 sys_catalog.cc:455] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 3ea36a1a750c40b98e9e225442fadbd1. Latest consensus state: current_term: 3 leader_uuid: "3ea36a1a750c40b98e9e225442fadbd1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3ea36a1a750c40b98e9e225442fadbd1" member_type: VOTER last_known_addr { host: "127.19.180.62" port: 46081 } } peers { permanent_uuid: "ff11b88709b14cda87f5cecd80a792d6" member_type: VOTER last_known_addr { host: "127.19.180.61" port: 37725 } } peers { permanent_uuid: "528e2c60e3bc4a5aa2ba21cac324d896" member_type: VOTER last_known_addr { host: "127.19.180.60" port: 32797 } } }
15:02:54.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.532205 20998 sys_catalog.cc:458] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [sys.catalog]: This master's current role is: FOLLOWER
15:02:54.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.533568 20986 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "3ea36a1a750c40b98e9e225442fadbd1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3ea36a1a750c40b98e9e225442fadbd1" member_type: VOTER last_known_addr { host: "127.19.180.62" port: 46081 } } peers { permanent_uuid: "ff11b88709b14cda87f5cecd80a792d6" member_type: VOTER last_known_addr { host: "127.19.180.61" port: 37725 } } peers { permanent_uuid: "528e2c60e3bc4a5aa2ba21cac324d896" member_type: VOTER last_known_addr { host: "127.19.180.60" port: 32797 } } }
15:02:54.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.533910 20986 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1 [sys.catalog]: This master's current role is: LEADER
15:02:54.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.534673 20998 sys_catalog.cc:455] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "3ea36a1a750c40b98e9e225442fadbd1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3ea36a1a750c40b98e9e225442fadbd1" member_type: VOTER last_known_addr { host: "127.19.180.62" port: 46081 } } peers { permanent_uuid: "ff11b88709b14cda87f5cecd80a792d6" member_type: VOTER last_known_addr { host: "127.19.180.61" port: 37725 } } peers { permanent_uuid: "528e2c60e3bc4a5aa2ba21cac324d896" member_type: VOTER last_known_addr { host: "127.19.180.60" port: 32797 } } }
15:02:54.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.535041 20998 sys_catalog.cc:458] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [sys.catalog]: This master's current role is: FOLLOWER
15:02:54.547 [DEBUG - pool-38-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=9, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: OK, [3ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: OK, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: OK, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: OK, [123ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [123ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [143ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [145ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: OK, [145ms] delaying RPC due to: Service unavailable: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [243ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: OK, [245ms] sending RPC to server master-127.19.180.62:46081, deferred=Deferred@60665556(state=PENDING, result=null, callback=(continuation of Deferred@1129259573 after retry RPC after error@1589893570) -> (continuation of Deferred@486665818 after retry RPC after error@1938032672) -> (continuation of Deferred@1231082223 after retry RPC after error@162336712) -> (continuation of Deferred@466371481 after retry RPC after error@1213894088) -> (continuation of Deferred@123918726 after retry RPC after error@1724929517) -> (continuation of Deferred@528028197 after retry RPC after error@64092367) -> (continuation of Deferred@317636570 after retry RPC after error@1995829079) -> (continuation of Deferred@1815773441 after retry RPC@511144300), errback=(continuation of Deferred@1129259573 after retry RPC after error@1589893570) -> (continuation of Deferred@486665818 after retry RPC after error@1938032672) -> (continuation of Deferred@1231082223 after retry RPC after error@162336712) -> (continuation of Deferred@466371481 after retry RPC after error@1213894088) -> (continuation of Deferred@123918726 after retry RPC after error@1724929517) -> (continuation of Deferred@528028197 after retry RPC after error@64092367) -> (continuation of Deferred@317636570 after retry RPC after error@1995829079) -> (continuation of Deferred@1815773441 after retry RPC@511144300)))
15:02:54.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:54.555325 20248 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:51392 (request call id 15) took 22 ms (client timeout 6 ms). Trace:
15:02:54.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:54.555546 20248 rpcz_store.cc:269] 0112 15:02:54.532918 (+ 0us) service_pool.cc:168] Inserting onto call queue
15:02:54.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:02:54.532997 (+ 79us) service_pool.cc:225] Handling call
15:02:54.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:02:54.555232 (+ 22235us) inbound_call.cc:173] Queueing success response
15:02:54.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"init.queue_time_us":1013,"thread_start_us":1714,"threads_started":3}
15:02:54.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/monotime.cc:332:28: runtime error: signed integer overflow: 849808926363 + 9223371187045850743 cannot be represented in type 'long'
15:02:54.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.634553 20217 catalog_manager.cc:1909] Servicing CreateTable request from {username='slave'} at 127.0.0.1:51410:
15:02:54.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
15:02:54.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
15:02:54.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:02:54.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
15:02:54.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
15:02:54.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
15:02:54.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
15:02:54.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
15:02:54.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
15:02:54.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
15:02:54.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
15:02:54.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:02:54.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:02:54.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
15:02:54.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
15:02:54.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
15:02:54.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
15:02:54.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
15:02:54.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
15:02:54.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
15:02:54.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
15:02:54.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:02:54.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:02:54.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
15:02:54.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
15:02:54.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
15:02:54.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
15:02:54.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
15:02:54.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
15:02:54.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
15:02:54.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
15:02:54.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:02:54.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:02:54.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
15:02:54.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
15:02:54.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
15:02:54.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
15:02:54.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
15:02:54.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
15:02:54.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
15:02:54.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
15:02:54.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:02:54.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:02:54.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
15:02:54.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
15:02:54.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
15:02:54.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
15:02:54.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:02:54.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
15:02:54.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
15:02:54.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:02:54.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
15:02:54.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:02:54.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:02:54.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:02:54.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
15:02:54.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:54.636309 20505 txn_status_manager.cc:1391] automatically aborted stale txn (ID 0) past 1.066s from last keepalive heartbeat (effective timeout is 1.000s)
15:02:54.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:54.638664 20217 catalog_manager.cc:6885] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
15:02:54.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:54.649942 20550 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.19.180.61:37725)
15:02:54.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:54.657249 21010 tablet_replica.cc:1307] Aborted: operation has been aborted: cancelling pending write operations
15:02:54.675 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) terminating keepalive task (txn ID 0) due to exception org.apache.kudu.client.NonRecoverableException: Failed to write to server: a0258fb56e9d4b858dfa55326a152559 (127.19.180.1:46735): transaction ID 0 is not available for further commits: state: ABORT_IN_PROGRESS user: "slave" start_timestamp: 1736694173 last_transition_timestamp: 1736694174
15:02:55.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:55.020486 20203 consensus_peers.cc:487] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1 -> Peer ff11b88709b14cda87f5cecd80a792d6 (127.19.180.61:37725): Couldn't send request to peer ff11b88709b14cda87f5cecd80a792d6. Status: Network error: Client connection negotiation failed: client connection to 127.19.180.61:37725: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
15:02:55.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:55.229354 20203 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.180.61:37725: connect: Connection refused (error 111) [suppressed 17 similar messages]
15:02:55.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:55.492616 20203 consensus_peers.cc:487] T 00000000000000000000000000000000 P 3ea36a1a750c40b98e9e225442fadbd1 -> Peer ff11b88709b14cda87f5cecd80a792d6 (127.19.180.61:37725): Couldn't send request to peer ff11b88709b14cda87f5cecd80a792d6. Status: Network error: Client connection negotiation failed: client connection to 127.19.180.61:37725: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
15:02:55.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 0x7fee6e8e5b08 in kudu::operator+(kudu::MonoTime const&, kudu::MonoDelta const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/monotime.cc:332:28
15:02:55.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 0xa614e5 in kudu::CountDownLatch::WaitFor(kudu::MonoDelta const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/countdown_latch.h:99:38
15:02:55.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:55.616578 20764 heartbeater.cc:502] Master 127.19.180.62:46081 was elected leader, sending a full tablet report...
15:02:55.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:55.620926 20623 heartbeater.cc:502] Master 127.19.180.62:46081 was elected leader, sending a full tablet report...
15:02:55.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 0x7fee81e2ba32 in kudu::Promise<kudu::Status>::WaitFor(kudu::MonoDelta const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/promise.h:57:16
15:02:55.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 0x7fee81e1c204 in kudu::master::Master::WaitForTxnManagerInit(kudu::MonoTime) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/master.cc:458:48
15:02:55.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 0x7fee81f75b3e in kudu::transactions::TxnManager::CheckInitialized(kudu::MonoTime const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/txn_manager.cc:283:21
15:02:55.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 0x7fee81f7671b in kudu::transactions::TxnManager::KeepTransactionAlive(long, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, kudu::MonoTime const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/txn_manager.cc:226:3
15:02:55.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 0x7fee81f7b38f in kudu::transactions::TxnManagerServiceImpl::KeepTransactionAlive(kudu::transactions::KeepTransactionAliveRequestPB const*, kudu::transactions::KeepTransactionAliveResponsePB*, kudu::rpc::RpcContext*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/txn_manager_service.cc:159:42
15:02:55.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 0x7fee7bf37aa6 in kudu::transactions::TxnManagerServiceIf::TxnManagerServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_9::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/build/asan/src/kudu/master/txn_manager.service.cc:271:13
15:02:55.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 0x7fee7bf377d5 in std::_Function_handler<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*), kudu::transactions::TxnManagerServiceIf::TxnManagerServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_9>::_M_invoke(std::_Any_data const&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) ../../../include/c++/7.5.0/bits/std_function.h:316:2
15:02:55.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 0x7fee756f230e in std::function<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const ../../../include/c++/7.5.0/bits/std_function.h:706:14
15:02:55.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 0x7fee756f11ee in kudu::rpc::GeneratedServiceIf::Handle(kudu::rpc::InboundCall*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_if.cc:137:3
15:02:55.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 0x7fee756f7425 in kudu::rpc::ServicePool::RunThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_pool.cc:229:15
15:02:55.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 0x7fee756f7d4f in kudu::rpc::ServicePool::Init(int)::$_0::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_pool.cc:92:5
15:02:55.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 0x7fee756f7b78 in std::_Function_handler<void (), kudu::rpc::ServicePool::Init(int)::$_0>::_M_invoke(std::_Any_data const&) ../../../include/c++/7.5.0/bits/std_function.h:316:2
15:02:55.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 0xa71ecc in std::function<void ()>::operator()() const ../../../include/c++/7.5.0/bits/std_function.h:706:14
15:02:55.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 0x7fee6e9b00bd in kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:693:3
15:02:55.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 0x7fee71d936da in start_thread (/lib/x86_64-linux-gnu/libpthread.so.0+0x76da)
15:02:55.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 0x7fee6ab0371e in clone (/lib/x86_64-linux-gnu/libc.so.6+0x12171e)
15:02:55.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:02:55.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) SUMMARY: UndefinedBehaviorSanitizer: undefined-behavior /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/monotime.cc:332:28 in
15:02:55.690 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.19.180.62:46081(127.19.180.62:46081)] handling channelInactive
15:02:55.690 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.180.62:46081(127.19.180.62:46081)] cleaning up while in state READY due to: connection closed
15:02:55.690 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location master-127.19.180.62:46081(127.19.180.62:46081) for tablet Kudu Master: connection closed
15:02:55.708 [WARN - pool-38-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
15:02:55.720 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.19.180.62:46081(127.19.180.62:46081)] connecting to peer
15:02:55.721 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.19.180.61:37725(127.19.180.61:37725)] connecting to peer
15:02:55.721 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:02:55.722 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.180.62:46081(127.19.180.62:46081)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:02:55.722 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.62:46081: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:02:55.722 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:55.722 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.180.61:37725(127.19.180.61:37725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:55.722 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.61:37725: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:55.723 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797; will retry
15:02:55.728 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.62:46081(127.19.180.62:46081)] connecting to peer
15:02:55.728 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.61:37725(127.19.180.61:37725)] connecting to peer
15:02:55.728 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:02:55.728 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.180.62:46081(127.19.180.62:46081)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:02:55.728 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.62:46081: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:02:55.729 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:55.729 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.180.61:37725(127.19.180.61:37725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:55.729 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.61:37725: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:55.730 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797; will retry
15:02:55.747 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.62:46081(127.19.180.62:46081)] connecting to peer
15:02:55.748 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.61:37725(127.19.180.61:37725)] connecting to peer
15:02:55.748 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:02:55.748 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.180.62:46081(127.19.180.62:46081)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:02:55.749 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.62:46081: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:02:55.749 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:55.749 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.180.61:37725(127.19.180.61:37725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:55.749 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.61:37725: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:55.750 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797; will retry
15:02:55.774 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.62:46081(127.19.180.62:46081)] connecting to peer
15:02:55.775 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.61:37725(127.19.180.61:37725)] connecting to peer
15:02:55.775 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:02:55.775 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.180.62:46081(127.19.180.62:46081)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:02:55.775 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.62:46081: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:02:55.775 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:55.776 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.180.61:37725(127.19.180.61:37725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:55.776 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.61:37725: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:55.777 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797; will retry
15:02:55.787 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.62:46081(127.19.180.62:46081)] connecting to peer
15:02:55.788 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.61:37725(127.19.180.61:37725)] connecting to peer
15:02:55.788 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:02:55.788 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.180.62:46081(127.19.180.62:46081)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:02:55.788 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.62:46081: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:02:55.789 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:55.789 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.180.61:37725(127.19.180.61:37725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:55.789 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.61:37725: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:55.790 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797; will retry
15:02:55.807 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.62:46081(127.19.180.62:46081)] connecting to peer
15:02:55.808 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.61:37725(127.19.180.61:37725)] connecting to peer
15:02:55.808 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:02:55.808 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.180.62:46081(127.19.180.62:46081)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:02:55.808 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.62:46081: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:02:55.808 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:55.808 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.180.61:37725(127.19.180.61:37725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:55.809 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.61:37725: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:55.809 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797; will retry
15:02:55.847 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.62:46081(127.19.180.62:46081)] connecting to peer
15:02:55.848 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.61:37725(127.19.180.61:37725)] connecting to peer
15:02:55.848 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:02:55.849 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.180.62:46081(127.19.180.62:46081)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:02:55.849 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.62:46081: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:02:55.849 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:55.849 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.180.61:37725(127.19.180.61:37725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:55.849 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.61:37725: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:55.850 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797; will retry
15:02:55.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:55.899782 21036 raft_consensus.cc:491] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 3ea36a1a750c40b98e9e225442fadbd1)
15:02:55.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:55.899995 21036 raft_consensus.cc:513] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3ea36a1a750c40b98e9e225442fadbd1" member_type: VOTER last_known_addr { host: "127.19.180.62" port: 46081 } } peers { permanent_uuid: "ff11b88709b14cda87f5cecd80a792d6" member_type: VOTER last_known_addr { host: "127.19.180.61" port: 37725 } } peers { permanent_uuid: "528e2c60e3bc4a5aa2ba21cac324d896" member_type: VOTER last_known_addr { host: "127.19.180.60" port: 32797 } }
15:02:55.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:55.900780 21036 leader_election.cc:290] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 3ea36a1a750c40b98e9e225442fadbd1 (127.19.180.62:46081), ff11b88709b14cda87f5cecd80a792d6 (127.19.180.61:37725)
15:02:55.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:55.902755 20367 leader_election.cc:336] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 3ea36a1a750c40b98e9e225442fadbd1 (127.19.180.62:46081): Network error: Client connection negotiation failed: client connection to 127.19.180.62:46081: connect: Connection refused (error 111)
15:02:55.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:55.902923 20365 leader_election.cc:336] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer ff11b88709b14cda87f5cecd80a792d6 (127.19.180.61:37725): Network error: Client connection negotiation failed: client connection to 127.19.180.61:37725: connect: Connection refused (error 111)
15:02:55.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:55.903082 20365 leader_election.cc:304] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 528e2c60e3bc4a5aa2ba21cac324d896; no voters: 3ea36a1a750c40b98e9e225442fadbd1, ff11b88709b14cda87f5cecd80a792d6
15:02:55.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:55.903363 21036 raft_consensus.cc:2743] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
15:02:55.947 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.62:46081(127.19.180.62:46081)] connecting to peer
15:02:55.948 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.61:37725(127.19.180.61:37725)] connecting to peer
15:02:55.948 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:02:55.948 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.180.62:46081(127.19.180.62:46081)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:02:55.948 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:55.948 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.62:46081: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:02:55.949 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.180.61:37725(127.19.180.61:37725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:55.949 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.61:37725: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:55.950 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797; will retry
15:02:55.987 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.62:46081(127.19.180.62:46081)] connecting to peer
15:02:55.988 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.61:37725(127.19.180.61:37725)] connecting to peer
15:02:55.988 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:02:55.988 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.180.62:46081(127.19.180.62:46081)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:02:55.989 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.62:46081: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:02:55.989 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:55.989 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.180.61:37725(127.19.180.61:37725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:55.989 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.61:37725: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:55.990 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797; will retry
15:02:56.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:56.069025 20905 heartbeater.cc:502] Master 127.19.180.62:46081 was elected leader, sending a full tablet report...
15:02:56.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:56.263530 21036 raft_consensus.cc:491] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 3ea36a1a750c40b98e9e225442fadbd1)
15:02:56.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:56.263725 21036 raft_consensus.cc:513] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3ea36a1a750c40b98e9e225442fadbd1" member_type: VOTER last_known_addr { host: "127.19.180.62" port: 46081 } } peers { permanent_uuid: "ff11b88709b14cda87f5cecd80a792d6" member_type: VOTER last_known_addr { host: "127.19.180.61" port: 37725 } } peers { permanent_uuid: "528e2c60e3bc4a5aa2ba21cac324d896" member_type: VOTER last_known_addr { host: "127.19.180.60" port: 32797 } }
15:02:56.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:56.264384 21036 leader_election.cc:290] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 3ea36a1a750c40b98e9e225442fadbd1 (127.19.180.62:46081), ff11b88709b14cda87f5cecd80a792d6 (127.19.180.61:37725)
15:02:56.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:56.265591 20367 leader_election.cc:336] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 3ea36a1a750c40b98e9e225442fadbd1 (127.19.180.62:46081): Network error: Client connection negotiation failed: client connection to 127.19.180.62:46081: connect: Connection refused (error 111)
15:02:56.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:56.265794 20365 leader_election.cc:336] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer ff11b88709b14cda87f5cecd80a792d6 (127.19.180.61:37725): Network error: Client connection negotiation failed: client connection to 127.19.180.61:37725: connect: Connection refused (error 111)
15:02:56.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:56.265939 20365 leader_election.cc:304] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 528e2c60e3bc4a5aa2ba21cac324d896; no voters: 3ea36a1a750c40b98e9e225442fadbd1, ff11b88709b14cda87f5cecd80a792d6
15:02:56.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:56.266211 21036 raft_consensus.cc:2743] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
15:02:56.307 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.62:46081(127.19.180.62:46081)] connecting to peer
15:02:56.308 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.61:37725(127.19.180.61:37725)] connecting to peer
15:02:56.308 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:02:56.308 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.180.62:46081(127.19.180.62:46081)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:02:56.308 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.62:46081: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:02:56.309 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:56.309 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.180.61:37725(127.19.180.61:37725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:56.309 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.61:37725: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:56.310 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797; will retry
15:02:56.647 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.62:46081(127.19.180.62:46081)] connecting to peer
15:02:56.647 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.61:37725(127.19.180.61:37725)] connecting to peer
15:02:56.648 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:02:56.648 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.180.62:46081(127.19.180.62:46081)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:02:56.648 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.62:46081: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:02:56.648 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:56.648 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.180.61:37725(127.19.180.61:37725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:56.649 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.61:37725: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:56.649 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797; will retry
15:02:56.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:56.771574 21044 raft_consensus.cc:491] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 3ea36a1a750c40b98e9e225442fadbd1)
15:02:56.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:56.771776 21044 raft_consensus.cc:513] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3ea36a1a750c40b98e9e225442fadbd1" member_type: VOTER last_known_addr { host: "127.19.180.62" port: 46081 } } peers { permanent_uuid: "ff11b88709b14cda87f5cecd80a792d6" member_type: VOTER last_known_addr { host: "127.19.180.61" port: 37725 } } peers { permanent_uuid: "528e2c60e3bc4a5aa2ba21cac324d896" member_type: VOTER last_known_addr { host: "127.19.180.60" port: 32797 } }
15:02:56.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:56.772377 21044 leader_election.cc:290] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 3ea36a1a750c40b98e9e225442fadbd1 (127.19.180.62:46081), ff11b88709b14cda87f5cecd80a792d6 (127.19.180.61:37725)
15:02:56.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:56.776171 20365 leader_election.cc:336] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer ff11b88709b14cda87f5cecd80a792d6 (127.19.180.61:37725): Network error: Client connection negotiation failed: client connection to 127.19.180.61:37725: connect: Connection refused (error 111)
15:02:56.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:56.776378 20367 leader_election.cc:336] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 3ea36a1a750c40b98e9e225442fadbd1 (127.19.180.62:46081): Network error: Client connection negotiation failed: client connection to 127.19.180.62:46081: connect: Connection refused (error 111)
15:02:56.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:56.776489 20367 leader_election.cc:304] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 528e2c60e3bc4a5aa2ba21cac324d896; no voters: 3ea36a1a750c40b98e9e225442fadbd1, ff11b88709b14cda87f5cecd80a792d6
15:02:56.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:56.776717 21044 raft_consensus.cc:2743] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
15:02:57.067 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.62:46081(127.19.180.62:46081)] connecting to peer
15:02:57.068 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.61:37725(127.19.180.61:37725)] connecting to peer
15:02:57.068 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:02:57.068 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.180.62:46081(127.19.180.62:46081)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:02:57.068 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.62:46081: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:02:57.069 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:57.069 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.180.61:37725(127.19.180.61:37725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:57.069 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.61:37725: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:57.070 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797; will retry
15:02:57.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:57.408898 21050 raft_consensus.cc:491] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 3ea36a1a750c40b98e9e225442fadbd1)
15:02:57.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:57.409060 21050 raft_consensus.cc:513] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3ea36a1a750c40b98e9e225442fadbd1" member_type: VOTER last_known_addr { host: "127.19.180.62" port: 46081 } } peers { permanent_uuid: "ff11b88709b14cda87f5cecd80a792d6" member_type: VOTER last_known_addr { host: "127.19.180.61" port: 37725 } } peers { permanent_uuid: "528e2c60e3bc4a5aa2ba21cac324d896" member_type: VOTER last_known_addr { host: "127.19.180.60" port: 32797 } }
15:02:57.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:57.409664 21050 leader_election.cc:290] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 3ea36a1a750c40b98e9e225442fadbd1 (127.19.180.62:46081), ff11b88709b14cda87f5cecd80a792d6 (127.19.180.61:37725)
15:02:57.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:57.411543 20367 leader_election.cc:336] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 3ea36a1a750c40b98e9e225442fadbd1 (127.19.180.62:46081): Network error: Client connection negotiation failed: client connection to 127.19.180.62:46081: connect: Connection refused (error 111)
15:02:57.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:57.411742 20365 leader_election.cc:336] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer ff11b88709b14cda87f5cecd80a792d6 (127.19.180.61:37725): Network error: Client connection negotiation failed: client connection to 127.19.180.61:37725: connect: Connection refused (error 111)
15:02:57.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:57.411870 20365 leader_election.cc:304] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 528e2c60e3bc4a5aa2ba21cac324d896; no voters: 3ea36a1a750c40b98e9e225442fadbd1, ff11b88709b14cda87f5cecd80a792d6
15:02:57.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:57.412099 21050 raft_consensus.cc:2743] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
15:02:57.728 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.62:46081(127.19.180.62:46081)] connecting to peer
15:02:57.728 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.61:37725(127.19.180.61:37725)] connecting to peer
15:02:57.729 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:02:57.729 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.180.62:46081(127.19.180.62:46081)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:02:57.729 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.62:46081: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:02:57.729 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:57.729 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.180.61:37725(127.19.180.61:37725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:57.729 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.61:37725: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:57.730 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797; will retry
15:02:57.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:57.752461 21050 raft_consensus.cc:491] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 3ea36a1a750c40b98e9e225442fadbd1)
15:02:57.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:57.752635 21050 raft_consensus.cc:513] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3ea36a1a750c40b98e9e225442fadbd1" member_type: VOTER last_known_addr { host: "127.19.180.62" port: 46081 } } peers { permanent_uuid: "ff11b88709b14cda87f5cecd80a792d6" member_type: VOTER last_known_addr { host: "127.19.180.61" port: 37725 } } peers { permanent_uuid: "528e2c60e3bc4a5aa2ba21cac324d896" member_type: VOTER last_known_addr { host: "127.19.180.60" port: 32797 } }
15:02:57.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:57.753175 21050 leader_election.cc:290] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 3ea36a1a750c40b98e9e225442fadbd1 (127.19.180.62:46081), ff11b88709b14cda87f5cecd80a792d6 (127.19.180.61:37725)
15:02:57.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:57.754412 20365 leader_election.cc:336] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer ff11b88709b14cda87f5cecd80a792d6 (127.19.180.61:37725): Network error: Client connection negotiation failed: client connection to 127.19.180.61:37725: connect: Connection refused (error 111)
15:02:57.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:57.754588 20367 leader_election.cc:336] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 3ea36a1a750c40b98e9e225442fadbd1 (127.19.180.62:46081): Network error: Client connection negotiation failed: client connection to 127.19.180.62:46081: connect: Connection refused (error 111)
15:02:57.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:57.754710 20367 leader_election.cc:304] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 528e2c60e3bc4a5aa2ba21cac324d896; no voters: 3ea36a1a750c40b98e9e225442fadbd1, ff11b88709b14cda87f5cecd80a792d6
15:02:57.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:57.754931 21050 raft_consensus.cc:2743] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
15:02:59.107 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.62:46081(127.19.180.62:46081)] connecting to peer
15:02:59.108 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.61:37725(127.19.180.61:37725)] connecting to peer
15:02:59.108 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:02:59.108 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.180.62:46081(127.19.180.62:46081)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:02:59.108 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.62:46081: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:02:59.109 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:59.109 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.180.61:37725(127.19.180.61:37725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:59.109 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.61:37725: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:02:59.110 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797; will retry
15:02:59.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:59.695346 21065 raft_consensus.cc:491] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 3ea36a1a750c40b98e9e225442fadbd1)
15:02:59.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:59.695531 21065 raft_consensus.cc:513] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3ea36a1a750c40b98e9e225442fadbd1" member_type: VOTER last_known_addr { host: "127.19.180.62" port: 46081 } } peers { permanent_uuid: "ff11b88709b14cda87f5cecd80a792d6" member_type: VOTER last_known_addr { host: "127.19.180.61" port: 37725 } } peers { permanent_uuid: "528e2c60e3bc4a5aa2ba21cac324d896" member_type: VOTER last_known_addr { host: "127.19.180.60" port: 32797 } }
15:02:59.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:59.696163 21065 leader_election.cc:290] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 3ea36a1a750c40b98e9e225442fadbd1 (127.19.180.62:46081), ff11b88709b14cda87f5cecd80a792d6 (127.19.180.61:37725)
15:02:59.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:59.697000 20367 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.180.62:46081: connect: Connection refused (error 111) [suppressed 11 similar messages]
15:02:59.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:59.698164 20367 leader_election.cc:336] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 3ea36a1a750c40b98e9e225442fadbd1 (127.19.180.62:46081): Network error: Client connection negotiation failed: client connection to 127.19.180.62:46081: connect: Connection refused (error 111)
15:02:59.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:02:59.698347 20365 leader_election.cc:336] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer ff11b88709b14cda87f5cecd80a792d6 (127.19.180.61:37725): Network error: Client connection negotiation failed: client connection to 127.19.180.61:37725: connect: Connection refused (error 111)
15:02:59.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:59.698441 20365 leader_election.cc:304] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 528e2c60e3bc4a5aa2ba21cac324d896; no voters: 3ea36a1a750c40b98e9e225442fadbd1, ff11b88709b14cda87f5cecd80a792d6
15:02:59.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:02:59.698647 21065 raft_consensus.cc:2743] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
15:03:01.668 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.62:46081(127.19.180.62:46081)] connecting to peer
15:03:01.668 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.61:37725(127.19.180.61:37725)] connecting to peer
15:03:01.669 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:01.669 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.180.62:46081(127.19.180.62:46081)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:01.669 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.62:46081: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:01.669 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:01.669 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.180.61:37725(127.19.180.61:37725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:01.669 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.61:37725: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:01.670 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797; will retry
15:03:02.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:03:02.252990 21079 raft_consensus.cc:491] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 3ea36a1a750c40b98e9e225442fadbd1)
15:03:02.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:03:02.253178 21079 raft_consensus.cc:513] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3ea36a1a750c40b98e9e225442fadbd1" member_type: VOTER last_known_addr { host: "127.19.180.62" port: 46081 } } peers { permanent_uuid: "ff11b88709b14cda87f5cecd80a792d6" member_type: VOTER last_known_addr { host: "127.19.180.61" port: 37725 } } peers { permanent_uuid: "528e2c60e3bc4a5aa2ba21cac324d896" member_type: VOTER last_known_addr { host: "127.19.180.60" port: 32797 } }
15:03:02.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:03:02.253813 21079 leader_election.cc:290] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 3ea36a1a750c40b98e9e225442fadbd1 (127.19.180.62:46081), ff11b88709b14cda87f5cecd80a792d6 (127.19.180.61:37725)
15:03:02.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:03:02.255893 20367 leader_election.cc:336] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 3ea36a1a750c40b98e9e225442fadbd1 (127.19.180.62:46081): Network error: Client connection negotiation failed: client connection to 127.19.180.62:46081: connect: Connection refused (error 111)
15:03:02.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:03:02.256073 20365 leader_election.cc:336] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer ff11b88709b14cda87f5cecd80a792d6 (127.19.180.61:37725): Network error: Client connection negotiation failed: client connection to 127.19.180.61:37725: connect: Connection refused (error 111)
15:03:02.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:03:02.256208 20365 leader_election.cc:304] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 528e2c60e3bc4a5aa2ba21cac324d896; no voters: 3ea36a1a750c40b98e9e225442fadbd1, ff11b88709b14cda87f5cecd80a792d6
15:03:02.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:03:02.256486 21079 raft_consensus.cc:2743] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
15:03:03.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:03:03.027107 21085 raft_consensus.cc:491] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 3ea36a1a750c40b98e9e225442fadbd1)
15:03:03.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:03:03.027294 21085 raft_consensus.cc:513] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3ea36a1a750c40b98e9e225442fadbd1" member_type: VOTER last_known_addr { host: "127.19.180.62" port: 46081 } } peers { permanent_uuid: "ff11b88709b14cda87f5cecd80a792d6" member_type: VOTER last_known_addr { host: "127.19.180.61" port: 37725 } } peers { permanent_uuid: "528e2c60e3bc4a5aa2ba21cac324d896" member_type: VOTER last_known_addr { host: "127.19.180.60" port: 32797 } }
15:03:03.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:03:03.027894 21085 leader_election.cc:290] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 3ea36a1a750c40b98e9e225442fadbd1 (127.19.180.62:46081), ff11b88709b14cda87f5cecd80a792d6 (127.19.180.61:37725)
15:03:03.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:03:03.029775 20367 leader_election.cc:336] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 3ea36a1a750c40b98e9e225442fadbd1 (127.19.180.62:46081): Network error: Client connection negotiation failed: client connection to 127.19.180.62:46081: connect: Connection refused (error 111)
15:03:03.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:03:03.029927 20365 leader_election.cc:336] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer ff11b88709b14cda87f5cecd80a792d6 (127.19.180.61:37725): Network error: Client connection negotiation failed: client connection to 127.19.180.61:37725: connect: Connection refused (error 111)
15:03:03.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:03:03.030019 20365 leader_election.cc:304] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 528e2c60e3bc4a5aa2ba21cac324d896; no voters: 3ea36a1a750c40b98e9e225442fadbd1, ff11b88709b14cda87f5cecd80a792d6
15:03:03.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:03:03.030256 21085 raft_consensus.cc:2743] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
15:03:04.087 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.62:46081(127.19.180.62:46081)] connecting to peer
15:03:04.088 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.61:37725(127.19.180.61:37725)] connecting to peer
15:03:04.088 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:04.088 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.180.62:46081(127.19.180.62:46081)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:04.088 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.62:46081: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:04.088 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:04.089 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.180.61:37725(127.19.180.61:37725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:04.089 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.61:37725: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:04.090 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797; will retry
15:03:07.727 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.62:46081(127.19.180.62:46081)] connecting to peer
15:03:07.728 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.61:37725(127.19.180.61:37725)] connecting to peer
15:03:07.728 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:07.728 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.180.62:46081(127.19.180.62:46081)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:07.728 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.62:46081: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:07.729 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:07.729 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.180.61:37725(127.19.180.61:37725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:07.729 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.61:37725: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:07.730 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797; will retry
15:03:08.627 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.62:46081(127.19.180.62:46081)] connecting to peer
15:03:08.628 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.61:37725(127.19.180.61:37725)] connecting to peer
15:03:08.628 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:08.628 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.180.62:46081(127.19.180.62:46081)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:08.628 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.62:46081: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:08.629 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:08.629 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.180.61:37725(127.19.180.61:37725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:08.629 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.61:37725: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:08.630 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797; will retry
15:03:10.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:03:10.369864 21127 raft_consensus.cc:491] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 3ea36a1a750c40b98e9e225442fadbd1)
15:03:10.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:03:10.370079 21127 raft_consensus.cc:513] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3ea36a1a750c40b98e9e225442fadbd1" member_type: VOTER last_known_addr { host: "127.19.180.62" port: 46081 } } peers { permanent_uuid: "ff11b88709b14cda87f5cecd80a792d6" member_type: VOTER last_known_addr { host: "127.19.180.61" port: 37725 } } peers { permanent_uuid: "528e2c60e3bc4a5aa2ba21cac324d896" member_type: VOTER last_known_addr { host: "127.19.180.60" port: 32797 } }
15:03:10.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:03:10.370776 21127 leader_election.cc:290] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 3ea36a1a750c40b98e9e225442fadbd1 (127.19.180.62:46081), ff11b88709b14cda87f5cecd80a792d6 (127.19.180.61:37725)
15:03:10.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:03:10.371726 20365 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.180.61:37725: connect: Connection refused (error 111) [suppressed 5 similar messages]
15:03:10.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:03:10.372813 20365 leader_election.cc:336] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer ff11b88709b14cda87f5cecd80a792d6 (127.19.180.61:37725): Network error: Client connection negotiation failed: client connection to 127.19.180.61:37725: connect: Connection refused (error 111)
15:03:10.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:03:10.373001 20367 leader_election.cc:336] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 3ea36a1a750c40b98e9e225442fadbd1 (127.19.180.62:46081): Network error: Client connection negotiation failed: client connection to 127.19.180.62:46081: connect: Connection refused (error 111)
15:03:10.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:03:10.373131 20367 leader_election.cc:304] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 528e2c60e3bc4a5aa2ba21cac324d896; no voters: 3ea36a1a750c40b98e9e225442fadbd1, ff11b88709b14cda87f5cecd80a792d6
15:03:10.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:03:10.373364 21127 raft_consensus.cc:2743] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
15:03:12.208 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.62:46081(127.19.180.62:46081)] connecting to peer
15:03:12.208 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.61:37725(127.19.180.61:37725)] connecting to peer
15:03:12.209 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:12.209 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.180.62:46081(127.19.180.62:46081)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:12.209 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.62:46081: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:12.209 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:12.209 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.180.61:37725(127.19.180.61:37725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:12.209 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.61:37725: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:12.210 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797; will retry
15:03:15.167 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.62:46081(127.19.180.62:46081)] connecting to peer
15:03:15.168 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.61:37725(127.19.180.61:37725)] connecting to peer
15:03:15.168 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:15.169 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.180.62:46081(127.19.180.62:46081)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:15.169 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:15.169 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.180.61:37725(127.19.180.61:37725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:15.169 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.62:46081: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:15.169 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.61:37725: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:15.170 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797; will retry
15:03:15.408 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.62:46081(127.19.180.62:46081)] connecting to peer
15:03:15.408 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.61:37725(127.19.180.61:37725)] connecting to peer
15:03:15.408 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:15.409 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.180.62:46081(127.19.180.62:46081)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:15.409 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.62:46081: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:15.409 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:15.409 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.180.61:37725(127.19.180.61:37725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:15.409 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.61:37725: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:15.410 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797; will retry
15:03:16.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:03:16.003633 21161 raft_consensus.cc:491] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 3ea36a1a750c40b98e9e225442fadbd1)
15:03:16.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:03:16.003831 21161 raft_consensus.cc:513] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3ea36a1a750c40b98e9e225442fadbd1" member_type: VOTER last_known_addr { host: "127.19.180.62" port: 46081 } } peers { permanent_uuid: "ff11b88709b14cda87f5cecd80a792d6" member_type: VOTER last_known_addr { host: "127.19.180.61" port: 37725 } } peers { permanent_uuid: "528e2c60e3bc4a5aa2ba21cac324d896" member_type: VOTER last_known_addr { host: "127.19.180.60" port: 32797 } }
15:03:16.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:03:16.004458 21161 leader_election.cc:290] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 3ea36a1a750c40b98e9e225442fadbd1 (127.19.180.62:46081), ff11b88709b14cda87f5cecd80a792d6 (127.19.180.61:37725)
15:03:16.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:03:16.005262 20367 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.180.62:46081: connect: Connection refused (error 111) [suppressed 1 similar messages]
15:03:16.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:03:16.006529 20367 leader_election.cc:336] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 3ea36a1a750c40b98e9e225442fadbd1 (127.19.180.62:46081): Network error: Client connection negotiation failed: client connection to 127.19.180.62:46081: connect: Connection refused (error 111)
15:03:16.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:03:16.006680 20365 leader_election.cc:336] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer ff11b88709b14cda87f5cecd80a792d6 (127.19.180.61:37725): Network error: Client connection negotiation failed: client connection to 127.19.180.61:37725: connect: Connection refused (error 111)
15:03:16.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:03:16.006807 20365 leader_election.cc:304] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 528e2c60e3bc4a5aa2ba21cac324d896; no voters: 3ea36a1a750c40b98e9e225442fadbd1, ff11b88709b14cda87f5cecd80a792d6
15:03:16.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:03:16.007088 21161 raft_consensus.cc:2743] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
15:03:17.187 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.62:46081(127.19.180.62:46081)] connecting to peer
15:03:17.188 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.61:37725(127.19.180.61:37725)] connecting to peer
15:03:17.188 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:17.188 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.180.62:46081(127.19.180.62:46081)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:17.188 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.62:46081: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:17.188 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:17.189 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.180.61:37725(127.19.180.61:37725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:17.189 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.61:37725: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:17.190 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797; will retry
15:03:18.147 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.62:46081(127.19.180.62:46081)] connecting to peer
15:03:18.148 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.61:37725(127.19.180.61:37725)] connecting to peer
15:03:18.148 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:18.148 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.180.62:46081(127.19.180.62:46081)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:18.148 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.62:46081: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:18.149 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:18.149 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.180.61:37725(127.19.180.61:37725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:18.149 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.61:37725: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:18.150 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797; will retry
15:03:19.607 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.62:46081(127.19.180.62:46081)] connecting to peer
15:03:19.607 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.61:37725(127.19.180.61:37725)] connecting to peer
15:03:19.608 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:19.608 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.180.62:46081(127.19.180.62:46081)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:19.608 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.62:46081: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:19.608 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:19.608 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.180.61:37725(127.19.180.61:37725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:19.608 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.61:37725: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:19.610 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797; will retry
15:03:21.827 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.62:46081(127.19.180.62:46081)] connecting to peer
15:03:21.828 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.61:37725(127.19.180.61:37725)] connecting to peer
15:03:21.828 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:21.828 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.180.62:46081(127.19.180.62:46081)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:21.828 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.62:46081: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:21.828 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:21.829 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.180.61:37725(127.19.180.61:37725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:21.829 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.61:37725: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:21.830 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797; will retry
15:03:22.407 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.62:46081(127.19.180.62:46081)] connecting to peer
15:03:22.408 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.61:37725(127.19.180.61:37725)] connecting to peer
15:03:22.408 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:22.408 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.180.62:46081(127.19.180.62:46081)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:22.408 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:22.408 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.62:46081: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:22.408 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.180.61:37725(127.19.180.61:37725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:22.409 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.61:37725: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:22.409 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797; will retry
15:03:22.847 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.62:46081(127.19.180.62:46081)] connecting to peer
15:03:22.848 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.61:37725(127.19.180.61:37725)] connecting to peer
15:03:22.848 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:22.848 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.180.62:46081(127.19.180.62:46081)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:22.849 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.62:46081: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:22.849 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:22.849 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.180.61:37725(127.19.180.61:37725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:22.849 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.61:37725: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:22.850 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797; will retry
15:03:24.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:03:24.624853 21208 raft_consensus.cc:491] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 3ea36a1a750c40b98e9e225442fadbd1)
15:03:24.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:03:24.625053 21208 raft_consensus.cc:513] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3ea36a1a750c40b98e9e225442fadbd1" member_type: VOTER last_known_addr { host: "127.19.180.62" port: 46081 } } peers { permanent_uuid: "ff11b88709b14cda87f5cecd80a792d6" member_type: VOTER last_known_addr { host: "127.19.180.61" port: 37725 } } peers { permanent_uuid: "528e2c60e3bc4a5aa2ba21cac324d896" member_type: VOTER last_known_addr { host: "127.19.180.60" port: 32797 } }
15:03:24.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:03:24.625665 21208 leader_election.cc:290] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 3ea36a1a750c40b98e9e225442fadbd1 (127.19.180.62:46081), ff11b88709b14cda87f5cecd80a792d6 (127.19.180.61:37725)
15:03:24.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:03:24.626500 20367 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.180.62:46081: connect: Connection refused (error 111) [suppressed 1 similar messages]
15:03:24.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:03:24.628213 20367 leader_election.cc:336] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 3ea36a1a750c40b98e9e225442fadbd1 (127.19.180.62:46081): Network error: Client connection negotiation failed: client connection to 127.19.180.62:46081: connect: Connection refused (error 111)
15:03:24.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:03:24.628381 20365 leader_election.cc:336] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer ff11b88709b14cda87f5cecd80a792d6 (127.19.180.61:37725): Network error: Client connection negotiation failed: client connection to 127.19.180.61:37725: connect: Connection refused (error 111)
15:03:24.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:03:24.628472 20365 leader_election.cc:304] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 528e2c60e3bc4a5aa2ba21cac324d896; no voters: 3ea36a1a750c40b98e9e225442fadbd1, ff11b88709b14cda87f5cecd80a792d6
15:03:24.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:03:24.628692 21208 raft_consensus.cc:2743] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
15:03:26.127 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.62:46081(127.19.180.62:46081)] connecting to peer
15:03:26.128 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.61:37725(127.19.180.61:37725)] connecting to peer
15:03:26.128 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:26.128 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.180.62:46081(127.19.180.62:46081)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:26.129 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.62:46081: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:26.129 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:26.129 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.180.61:37725(127.19.180.61:37725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:26.129 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.61:37725: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:26.130 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797; will retry
15:03:27.088 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.62:46081(127.19.180.62:46081)] connecting to peer
15:03:27.088 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.61:37725(127.19.180.61:37725)] connecting to peer
15:03:27.088 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:27.089 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.180.62:46081(127.19.180.62:46081)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:27.089 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.62:46081: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:27.089 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:27.089 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.180.61:37725(127.19.180.61:37725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:27.089 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.61:37725: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:27.090 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797; will retry
15:03:29.327 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.62:46081(127.19.180.62:46081)] connecting to peer
15:03:29.328 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.61:37725(127.19.180.61:37725)] connecting to peer
15:03:29.328 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:29.329 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.180.62:46081(127.19.180.62:46081)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:29.329 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:29.329 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.62:46081: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:29.329 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.180.61:37725(127.19.180.61:37725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:29.329 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.61:37725: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:29.330 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797; will retry
15:03:30.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:03:30.530356 21242 raft_consensus.cc:491] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 3ea36a1a750c40b98e9e225442fadbd1)
15:03:30.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:03:30.530546 21242 raft_consensus.cc:513] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3ea36a1a750c40b98e9e225442fadbd1" member_type: VOTER last_known_addr { host: "127.19.180.62" port: 46081 } } peers { permanent_uuid: "ff11b88709b14cda87f5cecd80a792d6" member_type: VOTER last_known_addr { host: "127.19.180.61" port: 37725 } } peers { permanent_uuid: "528e2c60e3bc4a5aa2ba21cac324d896" member_type: VOTER last_known_addr { host: "127.19.180.60" port: 32797 } }
15:03:30.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:03:30.531152 21242 leader_election.cc:290] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 3ea36a1a750c40b98e9e225442fadbd1 (127.19.180.62:46081), ff11b88709b14cda87f5cecd80a792d6 (127.19.180.61:37725)
15:03:30.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:03:30.531931 20367 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.180.62:46081: connect: Connection refused (error 111) [suppressed 1 similar messages]
15:03:30.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:03:30.533185 20367 leader_election.cc:336] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 3ea36a1a750c40b98e9e225442fadbd1 (127.19.180.62:46081): Network error: Client connection negotiation failed: client connection to 127.19.180.62:46081: connect: Connection refused (error 111)
15:03:30.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:03:30.533356 20365 leader_election.cc:336] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer ff11b88709b14cda87f5cecd80a792d6 (127.19.180.61:37725): Network error: Client connection negotiation failed: client connection to 127.19.180.61:37725: connect: Connection refused (error 111)
15:03:30.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:03:30.533450 20365 leader_election.cc:304] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 528e2c60e3bc4a5aa2ba21cac324d896; no voters: 3ea36a1a750c40b98e9e225442fadbd1, ff11b88709b14cda87f5cecd80a792d6
15:03:30.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:03:30.533665 21242 raft_consensus.cc:2743] T 00000000000000000000000000000000 P 528e2c60e3bc4a5aa2ba21cac324d896 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
15:03:32.867 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.62:46081(127.19.180.62:46081)] connecting to peer
15:03:32.868 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.61:37725(127.19.180.61:37725)] connecting to peer
15:03:32.868 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:32.868 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.180.62:46081(127.19.180.62:46081)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:32.868 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.62:46081: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:32.869 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:32.869 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.180.61:37725(127.19.180.61:37725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:32.869 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.61:37725: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:32.870 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797; will retry
15:03:33.508 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.62:46081(127.19.180.62:46081)] connecting to peer
15:03:33.508 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.61:37725(127.19.180.61:37725)] connecting to peer
15:03:33.508 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:33.508 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.180.62:46081(127.19.180.62:46081)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:33.509 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.62:46081: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:33.509 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:33.509 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.180.61:37725(127.19.180.61:37725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:33.509 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.61:37725: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:33.510 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797; will retry
15:03:36.227 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.62:46081(127.19.180.62:46081)] connecting to peer
15:03:36.228 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.61:37725(127.19.180.61:37725)] connecting to peer
15:03:36.228 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:36.228 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.180.62:46081(127.19.180.62:46081)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:36.228 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.62:46081: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:36.228 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:36.228 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.180.61:37725(127.19.180.61:37725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:36.229 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.61:37725: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:36.230 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797; will retry
15:03:36.347 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.62:46081(127.19.180.62:46081)] connecting to peer
15:03:36.347 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.61:37725(127.19.180.61:37725)] connecting to peer
15:03:36.348 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:36.348 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.180.62:46081(127.19.180.62:46081)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:36.348 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.62:46081: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:36.348 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:36.348 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.180.61:37725(127.19.180.61:37725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:36.348 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.61:37725: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:36.349 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797; will retry
15:03:38.527 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.62:46081(127.19.180.62:46081)] connecting to peer
15:03:38.527 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.61:37725(127.19.180.61:37725)] connecting to peer
15:03:38.528 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:38.528 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.180.62:46081(127.19.180.62:46081)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:38.528 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.62:46081: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:38.528 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:38.528 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.180.61:37725(127.19.180.61:37725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:38.528 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.61:37725: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:38.530 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797; will retry
15:03:40.828 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.62:46081(127.19.180.62:46081)] connecting to peer
15:03:40.828 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.61:37725(127.19.180.61:37725)] connecting to peer
15:03:40.829 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:40.829 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.180.62:46081(127.19.180.62:46081)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:40.829 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.62:46081: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:40.829 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:40.829 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.180.61:37725(127.19.180.61:37725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:40.829 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.61:37725: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:40.830 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797; will retry
15:03:41.667 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.62:46081(127.19.180.62:46081)] connecting to peer
15:03:41.667 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.61:37725(127.19.180.61:37725)] connecting to peer
15:03:41.668 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:41.668 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.180.62:46081(127.19.180.62:46081)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:41.668 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.62:46081: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:41.668 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:41.668 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.180.61:37725(127.19.180.61:37725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:41.668 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.61:37725: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:41.669 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797; will retry
15:03:45.687 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.62:46081(127.19.180.62:46081)] connecting to peer
15:03:45.688 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.19.180.61:37725(127.19.180.61:37725)] connecting to peer
15:03:45.688 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:45.688 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.180.62:46081(127.19.180.62:46081)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:45.689 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.62:46081: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081
15:03:45.689 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:45.689 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.180.61:37725(127.19.180.61:37725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:45.689 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.180.61:37725: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
15:03:45.690 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797; will retry
15:03:45.691 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=CommitTransaction, tablet=null, attempt=38, TimeoutTracker(timeout=50000, elapsed=49969), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: Network error: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [3ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [7ms] refreshing cache from master, [8ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [8ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [8ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [8ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: Network error: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081, [9ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [9ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [10ms] delaying RPC due to: Service unavailable: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [27ms] refreshing cache from master, [27ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [28ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [28ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [28ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: Network error: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081, [29ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [29ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [30ms] delaying RPC due to: Service unavailable: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [47ms] refreshing cache from master, [47ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [55ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [55ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [55ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: Network error: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081, [56ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [56ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [57ms] delaying RPC due to: Service unavailable: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [67ms] refreshing cache from master, [67ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [68ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [68ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [68ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: Network error: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081, [69ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [69ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [70ms] delaying RPC due to: Service unavailable: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [87ms] refreshing cache from master, [87ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [88ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [88ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [88ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: Network error: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081, [89ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [89ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [89ms] delaying RPC due to: Service unavailable: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [127ms] refreshing cache from master, [127ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [128ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [128ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [129ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: Network error: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081, [129ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [130ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [130ms] delaying RPC due to: Service unavailable: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [227ms] refreshing cache from master, [227ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [228ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [228ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [228ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: Network error: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081, [229ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [229ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [230ms] delaying RPC due to: Service unavailable: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [267ms] refreshing cache from master, [267ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [268ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [268ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [268ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: Network error: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081, [269ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [269ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [270ms] delaying RPC due to: Service unavailable: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [587ms] refreshing cache from master, [587ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [588ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [588ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [588ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: Network error: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081, [589ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [589ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [590ms] delaying RPC due to: Service unavailable: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [927ms] refreshing cache from master, [927ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [927ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [928ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [928ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: Network error: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081, [928ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [929ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [930ms] delaying RPC due to: Service unavailable: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [1347ms] refreshing cache from master, [1347ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [1348ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [1348ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [1348ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: Network error: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081, [1349ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [1350ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [1350ms] delaying RPC due to: Service unavailable: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [2007ms] refreshing cache from master, [2008ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [2008ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [2009ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [2009ms] too many traces: truncated at 100 traces, deferred=Deferred@125470014(state=PENDING, result=null, callback=(continuation of Deferred@1451766105 after retry RPC after error@1162483984) -> (continuation of Deferred@327391233 after retry RPC after error@1082375226) -> (continuation of Deferred@1469468554 after retry RPC after error@1107386097) -> (continuation of Deferred@1296925984 after retry RPC after error@275940063) -> (continuation of Deferred@1971156292 after retry RPC after error@1036209100) -> (continuation of Deferred@478592465 after retry RPC after error@409383140) -> (continuation of Deferred@112144889 after retry RPC after error@563447755) -> (continuation of Deferred@139775517 after retry RPC after error@630767203) -> (continuation of Deferred@1523016656 after retry RPC after error@1465982108) -> (continuation of Deferred@2045449291 after retry RPC after error@425772723) -> (continuation of Deferred@2027487148 after retry RPC after error@654477783) -> (continuation of Deferred@116033584 after retry RPC after error@429953792) -> (continuation of Deferred@641350151 after retry RPC after error@356860710) -> (continuation of Deferred@1195172534 after retry RPC after error@1519997378) -> (continuation of Deferred@763671188 after retry RPC after error@317163633) -> (continuation of Deferred@854516169 after retry RPC after error@577347863) -> (continuation of Deferred@720480220 after retry RPC after error@83564275) -> (continuation of Deferred@1105092364 after retry RPC after error@785399609) -> (continuation of Deferred@154706552 after retry RPC after error@1437102370) -> (continuation of Deferred@1723687588 after retry RPC after error@1237037245) -> (continuation of Deferred@98365584 after retry RPC after error@334213224) -> (continuation of Deferred@516839667 after retry RPC after error@174337487) -> (continuation of Deferred@1759380579 after retry RPC after error@653820384) -> (continuation of Deferred@1243438092 after retry RPC after error@1631324885) -> (continuation of Deferred@2044323725 after retry RPC after error@458006231) -> (continuation of Deferred@990993550 after retry RPC after error@989496219) -> (continuation of Deferred@212463302 after retry RPC after error@1936483234) -> (continuation of Deferred@1118348397 after retry RPC after error@748683768) -> (continuation of Deferred@1874593140 after retry RPC after error@2136808018) -> (continuation of Deferred@2018595352 after retry RPC after error@1423057707) -> (continuation of Deferred@1531579232 after retry RPC after error@1280447953) -> (continuation of Deferred@1054953425 after retry RPC after error@2070935645) -> (continuation of Deferred@1895165275 after retry RPC after error@752194057) -> (continuation of Deferred@410963845 after retry RPC after error@483881415) -> (continuation of Deferred@1125636147 after retry RPC after error@1137856386) -> (continuation of Deferred@759083674 after retry RPC after error@180398003) -> (continuation of Deferred@2011898008 after retry RPC after error@638902200), errback=(continuation of Deferred@1451766105 after retry RPC after error@1162483984) -> (continuation of Deferred@327391233 after retry RPC after error@1082375226) -> (continuation of Deferred@1469468554 after retry RPC after error@1107386097) -> (continuation of Deferred@1296925984 after retry RPC after error@275940063) -> (continuation of Deferred@1971156292 after retry RPC after error@1036209100) -> (continuation of Deferred@478592465 after retry RPC after error@409383140) -> (continuation of Deferred@112144889 after retry RPC after error@563447755) -> (continuation of Deferred@139775517 after retry RPC after error@630767203) -> (continuation of Deferred@1523016656 after retry RPC after error@1465982108) -> (continuation of Deferred@2045449291 after retry RPC after error@425772723) -> (continuation of Deferred@2027487148 after retry RPC after error@654477783) -> (continuation of Deferred@116033584 after retry RPC after error@429953792) -> (continuation of Deferred@641350151 after retry RPC after error@356860710) -> (continuation of Deferred@1195172534 after retry RPC after error@1519997378) -> (continuation of Deferred@763671188 after retry RPC after error@317163633) -> (continuation of Deferred@854516169 after retry RPC after error@577347863) -> (continuation of Deferred@720480220 after retry RPC after error@83564275) -> (continuation of Deferred@1105092364 after retry RPC after error@785399609) -> (continuation of Deferred@154706552 after retry RPC after error@1437102370) -> (continuation of Deferred@1723687588 after retry RPC after error@1237037245) -> (continuation of Deferred@98365584 after retry RPC after error@334213224) -> (continuation of Deferred@516839667 after retry RPC after error@174337487) -> (continuation of Deferred@1759380579 after retry RPC after error@653820384) -> (continuation of Deferred@1243438092 after retry RPC after error@1631324885) -> (continuation of Deferred@2044323725 after retry RPC after error@458006231) -> (continuation of Deferred@990993550 after retry RPC after error@989496219) -> (continuation of Deferred@212463302 after retry RPC after error@1936483234) -> (continuation of Deferred@1118348397 after retry RPC after error@748683768) -> (continuation of Deferred@1874593140 after retry RPC after error@2136808018) -> (continuation of Deferred@2018595352 after retry RPC after error@1423057707) -> (continuation of Deferred@1531579232 after retry RPC after error@1280447953) -> (continuation of Deferred@1054953425 after retry RPC after error@2070935645) -> (continuation of Deferred@1895165275 after retry RPC after error@752194057) -> (continuation of Deferred@410963845 after retry RPC after error@483881415) -> (continuation of Deferred@1125636147 after retry RPC after error@1137856386) -> (continuation of Deferred@759083674 after retry RPC after error@180398003) -> (continuation of Deferred@2011898008 after retry RPC after error@638902200)))
15:03:45.693 [DEBUG - main] (Connection.java:575) [peer a0258fb56e9d4b858dfa55326a152559(127.19.180.1:46735)] disconnecting while in state READY
15:03:45.693 [DEBUG - main] (Connection.java:575) [peer master-127.19.180.62:46081(127.19.180.62:46081)] disconnecting while in state TERMINATED
15:03:45.693 [DEBUG - main] (Connection.java:575) [peer master-127.19.180.60:32797(127.19.180.60:32797)] disconnecting while in state READY
15:03:45.693 [DEBUG - main] (Connection.java:575) [peer master-127.19.180.61:37725(127.19.180.61:37725)] disconnecting while in state TERMINATED
15:03:45.694 [DEBUG - kudu-nio-1] (Connection.java:239) [peer a0258fb56e9d4b858dfa55326a152559(127.19.180.1:46735)] handling channelInactive
15:03:45.694 [DEBUG - kudu-nio-1] (Connection.java:709) [peer a0258fb56e9d4b858dfa55326a152559(127.19.180.1:46735)] cleaning up while in state READY due to: connection closed
15:03:45.694 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
15:03:45.695 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.19.180.60:32797(127.19.180.60:32797)] handling channelInactive
15:03:45.695 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.180.60:32797(127.19.180.60:32797)] cleaning up while in state READY due to: connection closed
15:03:45.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:03:45.695477 20176 external_mini_cluster.cc:1661] Attempting to check leaks for /tmp/dist-test-taskDlDeJ5/build/asan/bin/kudu pid 20484
15:03:45.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:03:45.706905 20617 generic_service.cc:196] Checking for leaks (request via RPC)
15:03:45.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:03:45.836889 20176 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskDlDeJ5/build/asan/bin/kudu with pid 20484
15:03:45.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:03:45.855942 20176 external_mini_cluster.cc:1661] Attempting to check leaks for /tmp/dist-test-taskDlDeJ5/build/asan/bin/kudu pid 20627
15:03:45.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:03:45.862118 20758 generic_service.cc:196] Checking for leaks (request via RPC)
15:03:45.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:03:45.972158 20176 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskDlDeJ5/build/asan/bin/kudu with pid 20627
15:03:45.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:03:45.987576 20783 connection.cc:537] client connection to 127.19.180.2:35451 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
15:03:45.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:03:45.987884 20176 external_mini_cluster.cc:1661] Attempting to check leaks for /tmp/dist-test-taskDlDeJ5/build/asan/bin/kudu pid 20768
15:03:45.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:03:45.987856 20783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
15:03:45.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:03:45.990185 20783 consensus_peers.cc:487] T 9727c0ccc99e478eb0c34096f44de1a6 P c45c25ab0a664200896372dcc7565f59 -> Peer 1ab780ab6f274ded9a5fdac3c28998c7 (127.19.180.2:35451): Couldn't send request to peer 1ab780ab6f274ded9a5fdac3c28998c7. Status: Network error: Client connection negotiation failed: client connection to 127.19.180.2:35451: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
15:03:45.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:03:45.994503 20899 generic_service.cc:196] Checking for leaks (request via RPC)
15:03:46.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:03:46.099159 20176 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskDlDeJ5/build/asan/bin/kudu with pid 20768
15:03:46.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:03:46.115100 20176 external_mini_cluster.cc:1661] Attempting to check leaks for /tmp/dist-test-taskDlDeJ5/build/asan/bin/kudu pid 20346
15:03:46.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:03:46.116187 20420 generic_service.cc:196] Checking for leaks (request via RPC)
15:03:46.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:03:46.230700 20176 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskDlDeJ5/build/asan/bin/kudu with pid 20346
15:03:46.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-12T15:03:46Z chronyd exiting
15:03:46.377 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveSwitchesToOtherTxnManager: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=CommitTransaction, tablet=null, attempt=38, TimeoutTracker(timeout=50000, elapsed=49969), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: Network error: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [3ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [7ms] refreshing cache from master, [8ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [8ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [8ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [8ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: Network error: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081, [9ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [9ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [10ms] delaying RPC due to: Service unavailable: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [27ms] refreshing cache from master, [27ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [28ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [28ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [28ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: Network error: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081, [29ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [29ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [30ms] delaying RPC due to: Service unavailable: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [47ms] refreshing cache from master, [47ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [55ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [55ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [55ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: Network error: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081, [56ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [56ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [57ms] delaying RPC due to: Service unavailable: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [67ms] refreshing cache from master, [67ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [68ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [68ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [68ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: Network error: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081, [69ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [69ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [70ms] delaying RPC due to: Service unavailable: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [87ms] refreshing cache from master, [87ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [88ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [88ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [88ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: Network error: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081, [89ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [89ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [89ms] delaying RPC due to: Service unavailable: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [127ms] refreshing cache from master, [127ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [128ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [128ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [129ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: Network error: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081, [129ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [130ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [130ms] delaying RPC due to: Service unavailable: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [227ms] refreshing cache from master, [227ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [228ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [228ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [228ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: Network error: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081, [229ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [229ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [230ms] delaying RPC due to: Service unavailable: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [267ms] refreshing cache from master, [267ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [268ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [268ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [268ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: Network error: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081, [269ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [269ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [270ms] delaying RPC due to: Service unavailable: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [587ms] refreshing cache from master, [587ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [588ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [588ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [588ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: Network error: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081, [589ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [589ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [590ms] delaying RPC due to: Service unavailable: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [927ms] refreshing cache from master, [927ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [927ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [928ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [928ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: Network error: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081, [928ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [929ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [930ms] delaying RPC due to: Service unavailable: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [1347ms] refreshing cache from master, [1347ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [1348ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [1348ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [1348ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: Network error: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081, [1349ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [1350ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [1350ms] delaying RPC due to: Service unavailable: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [2007ms] refreshing cache from master, [2008ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [2008ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [2009ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [2009ms] too many traces: truncated at 100 traces, deferred=Deferred@125470014(state=PENDING, result=null, callback=(continuation of Deferred@1451766105 after retry RPC after error@1162483984) -> (continuation of Deferred@327391233 after retry RPC after error@1082375226) -> (continuation of Deferred@1469468554 after retry RPC after error@1107386097) -> (continuation of Deferred@1296925984 after retry RPC after error@275940063) -> (continuation of Deferred@1971156292 after retry RPC after error@1036209100) -> (continuation of Deferred@478592465 after retry RPC after error@409383140) -> (continuation of Deferred@112144889 after retry RPC after error@563447755) -> (continuation of Deferred@139775517 after retry RPC after error@630767203) -> (continuation of Deferred@1523016656 after retry RPC after error@1465982108) -> (continuation of Deferred@2045449291 after retry RPC after error@425772723) -> (continuation of Deferred@2027487148 after retry RPC after error@654477783) -> (continuation of Deferred@116033584 after retry RPC after error@429953792) -> (continuation of Deferred@641350151 after retry RPC after error@356860710) -> (continuation of Deferred@1195172534 after retry RPC after error@1519997378) -> (continuation of Deferred@763671188 after retry RPC after error@317163633) -> (continuation of Deferred@854516169 after retry RPC after error@577347863) -> (continuation of Deferred@720480220 after retry RPC after error@83564275) -> (continuation of Deferred@1105092364 after retry RPC after error@785399609) -> (continuation of Deferred@154706552 after retry RPC after error@1437102370) -> (continuation of Deferred@1723687588 after retry RPC after error@1237037245) -> (continuation of Deferred@98365584 after retry RPC after error@334213224) -> (continuation of Deferred@516839667 after retry RPC after error@174337487) -> (continuation of Deferred@1759380579 after retry RPC after error@653820384) -> (continuation of Deferred@1243438092 after retry RPC after error@1631324885) -> (continuation of Deferred@2044323725 after retry RPC after error@458006231) -> (continuation of Deferred@990993550 after retry RPC after error@989496219) -> (continuation of Deferred@212463302 after retry RPC after error@1936483234) -> (continuation of Deferred@1118348397 after retry RPC after error@748683768) -> (continuation of Deferred@1874593140 after retry RPC after error@2136808018) -> (continuation of Deferred@2018595352 after retry RPC after error@1423057707) -> (continuation of Deferred@1531579232 after retry RPC after error@1280447953) -> (continuation of Deferred@1054953425 after retry RPC after error@2070935645) -> (continuation of Deferred@1895165275 after retry RPC after error@752194057) -> (continuation of Deferred@410963845 after retry RPC after error@483881415) -> (continuation of Deferred@1125636147 after retry RPC after error@1137856386) -> (continuation of Deferred@759083674 after retry RPC after error@180398003) -> (continuation of Deferred@2011898008 after retry RPC after error@638902200), errback=(continuation of Deferred@1451766105 after retry RPC after error@1162483984) -> (continuation of Deferred@327391233 after retry RPC after error@1082375226) -> (continuation of Deferred@1469468554 after retry RPC after error@1107386097) -> (continuation of Deferred@1296925984 after retry RPC after error@275940063) -> (continuation of Deferred@1971156292 after retry RPC after error@1036209100) -> (continuation of Deferred@478592465 after retry RPC after error@409383140) -> (continuation of Deferred@112144889 after retry RPC after error@563447755) -> (continuation of Deferred@139775517 after retry RPC after error@630767203) -> (continuation of Deferred@1523016656 after retry RPC after error@1465982108) -> (continuation of Deferred@2045449291 after retry RPC after error@425772723) -> (continuation of Deferred@2027487148 after retry RPC after error@654477783) -> (continuation of Deferred@116033584 after retry RPC after error@429953792) -> (continuation of Deferred@641350151 after retry RPC after error@356860710) -> (continuation of Deferred@1195172534 after retry RPC after error@1519997378) -> (continuation of Deferred@763671188 after retry RPC after error@317163633) -> (continuation of Deferred@854516169 after retry RPC after error@577347863) -> (continuation of Deferred@720480220 after retry RPC after error@83564275) -> (continuation of Deferred@1105092364 after retry RPC after error@785399609) -> (continuation of Deferred@154706552 after retry RPC after error@1437102370) -> (continuation of Deferred@1723687588 after retry RPC after error@1237037245) -> (continuation of Deferred@98365584 after retry RPC after error@334213224) -> (continuation of Deferred@516839667 after retry RPC after error@174337487) -> (continuation of Deferred@1759380579 after retry RPC after error@653820384) -> (continuation of Deferred@1243438092 after retry RPC after error@1631324885) -> (continuation of Deferred@2044323725 after retry RPC after error@458006231) -> (continuation of Deferred@990993550 after retry RPC after error@989496219) -> (continuation of Deferred@212463302 after retry RPC after error@1936483234) -> (continuation of Deferred@1118348397 after retry RPC after error@748683768) -> (continuation of Deferred@1874593140 after retry RPC after error@2136808018) -> (continuation of Deferred@2018595352 after retry RPC after error@1423057707) -> (continuation of Deferred@1531579232 after retry RPC after error@1280447953) -> (continuation of Deferred@1054953425 after retry RPC after error@2070935645) -> (continuation of Deferred@1895165275 after retry RPC after error@752194057) -> (continuation of Deferred@410963845 after retry RPC after error@483881415) -> (continuation of Deferred@1125636147 after retry RPC after error@1137856386) -> (continuation of Deferred@759083674 after retry RPC after error@180398003) -> (continuation of Deferred@2011898008 after retry RPC after error@638902200)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduTransaction.doCommitTransaction(KuduTransaction.java:491) ~[main/:?]
at org.apache.kudu.client.KuduTransaction.commitWithMode(KuduTransaction.java:537) ~[main/:?]
at org.apache.kudu.client.KuduTransaction.commit(KuduTransaction.java:284) ~[main/:?]
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveSwitchesToOtherTxnManager(TestKuduTransaction.java:1152) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1503) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1366) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1415) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1503) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1366) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1415) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:326) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:342) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.19.180.61:37725
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_275]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_275]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=CommitTransaction, tablet=null, attempt=38, TimeoutTracker(timeout=50000, elapsed=49969), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: Network error: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [3ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [7ms] refreshing cache from master, [8ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [8ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [8ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [8ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: Network error: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081, [9ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [9ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [10ms] delaying RPC due to: Service unavailable: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [27ms] refreshing cache from master, [27ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [28ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [28ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [28ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: Network error: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081, [29ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [29ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [30ms] delaying RPC due to: Service unavailable: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [47ms] refreshing cache from master, [47ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [55ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [55ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [55ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: Network error: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081, [56ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [56ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [57ms] delaying RPC due to: Service unavailable: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [67ms] refreshing cache from master, [67ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [68ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [68ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [68ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: Network error: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081, [69ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [69ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [70ms] delaying RPC due to: Service unavailable: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [87ms] refreshing cache from master, [87ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [88ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [88ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [88ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: Network error: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081, [89ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [89ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [89ms] delaying RPC due to: Service unavailable: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [127ms] refreshing cache from master, [127ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [128ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [128ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [129ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: Network error: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081, [129ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [130ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [130ms] delaying RPC due to: Service unavailable: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [227ms] refreshing cache from master, [227ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [228ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [228ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [228ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: Network error: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081, [229ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [229ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [230ms] delaying RPC due to: Service unavailable: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [267ms] refreshing cache from master, [267ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [268ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [268ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [268ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: Network error: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081, [269ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [269ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [270ms] delaying RPC due to: Service unavailable: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [587ms] refreshing cache from master, [587ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [588ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [588ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [588ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: Network error: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081, [589ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [589ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [590ms] delaying RPC due to: Service unavailable: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [927ms] refreshing cache from master, [927ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [927ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [928ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [928ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: Network error: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081, [928ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [929ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [930ms] delaying RPC due to: Service unavailable: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [1347ms] refreshing cache from master, [1347ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [1348ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [1348ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [1348ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.62:46081: Network error: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081, [1349ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.61:37725: Network error: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [1350ms] Sub RPC ConnectToMaster: received response from server master-127.19.180.60:32797: OK, [1350ms] delaying RPC due to: Service unavailable: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725, [2007ms] refreshing cache from master, [2008ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.62:46081, [2008ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.61:37725, [2009ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.180.60:32797, [2009ms] too many traces: truncated at 100 traces, deferred=Deferred@125470014(state=PENDING, result=null, callback=(continuation of Deferred@1451766105 after retry RPC after error@1162483984) -> (continuation of Deferred@327391233 after retry RPC after error@1082375226) -> (continuation of Deferred@1469468554 after retry RPC after error@1107386097) -> (continuation of Deferred@1296925984 after retry RPC after error@275940063) -> (continuation of Deferred@1971156292 after retry RPC after error@1036209100) -> (continuation of Deferred@478592465 after retry RPC after error@409383140) -> (continuation of Deferred@112144889 after retry RPC after error@563447755) -> (continuation of Deferred@139775517 after retry RPC after error@630767203) -> (continuation of Deferred@1523016656 after retry RPC after error@1465982108) -> (continuation of Deferred@2045449291 after retry RPC after error@425772723) -> (continuation of Deferred@2027487148 after retry RPC after error@654477783) -> (continuation of Deferred@116033584 after retry RPC after error@429953792) -> (continuation of Deferred@641350151 after retry RPC after error@356860710) -> (continuation of Deferred@1195172534 after retry RPC after error@1519997378) -> (continuation of Deferred@763671188 after retry RPC after error@317163633) -> (continuation of Deferred@854516169 after retry RPC after error@577347863) -> (continuation of Deferred@720480220 after retry RPC after error@83564275) -> (continuation of Deferred@1105092364 after retry RPC after error@785399609) -> (continuation of Deferred@154706552 after retry RPC after error@1437102370) -> (continuation of Deferred@1723687588 after retry RPC after error@1237037245) -> (continuation of Deferred@98365584 after retry RPC after error@334213224) -> (continuation of Deferred@516839667 after retry RPC after error@174337487) -> (continuation of Deferred@1759380579 after retry RPC after error@653820384) -> (continuation of Deferred@1243438092 after retry RPC after error@1631324885) -> (continuation of Deferred@2044323725 after retry RPC after error@458006231) -> (continuation of Deferred@990993550 after retry RPC after error@989496219) -> (continuation of Deferred@212463302 after retry RPC after error@1936483234) -> (continuation of Deferred@1118348397 after retry RPC after error@748683768) -> (continuation of Deferred@1874593140 after retry RPC after error@2136808018) -> (continuation of Deferred@2018595352 after retry RPC after error@1423057707) -> (continuation of Deferred@1531579232 after retry RPC after error@1280447953) -> (continuation of Deferred@1054953425 after retry RPC after error@2070935645) -> (continuation of Deferred@1895165275 after retry RPC after error@752194057) -> (continuation of Deferred@410963845 after retry RPC after error@483881415) -> (continuation of Deferred@1125636147 after retry RPC after error@1137856386) -> (continuation of Deferred@759083674 after retry RPC after error@180398003) -> (continuation of Deferred@2011898008 after retry RPC after error@638902200), errback=(continuation of Deferred@1451766105 after retry RPC after error@1162483984) -> (continuation of Deferred@327391233 after retry RPC after error@1082375226) -> (continuation of Deferred@1469468554 after retry RPC after error@1107386097) -> (continuation of Deferred@1296925984 after retry RPC after error@275940063) -> (continuation of Deferred@1971156292 after retry RPC after error@1036209100) -> (continuation of Deferred@478592465 after retry RPC after error@409383140) -> (continuation of Deferred@112144889 after retry RPC after error@563447755) -> (continuation of Deferred@139775517 after retry RPC after error@630767203) -> (continuation of Deferred@1523016656 after retry RPC after error@1465982108) -> (continuation of Deferred@2045449291 after retry RPC after error@425772723) -> (continuation of Deferred@2027487148 after retry RPC after error@654477783) -> (continuation of Deferred@116033584 after retry RPC after error@429953792) -> (continuation of Deferred@641350151 after retry RPC after error@356860710) -> (continuation of Deferred@1195172534 after retry RPC after error@1519997378) -> (continuation of Deferred@763671188 after retry RPC after error@317163633) -> (continuation of Deferred@854516169 after retry RPC after error@577347863) -> (continuation of Deferred@720480220 after retry RPC after error@83564275) -> (continuation of Deferred@1105092364 after retry RPC after error@785399609) -> (continuation of Deferred@154706552 after retry RPC after error@1437102370) -> (continuation of Deferred@1723687588 after retry RPC after error@1237037245) -> (continuation of Deferred@98365584 after retry RPC after error@334213224) -> (continuation of Deferred@516839667 after retry RPC after error@174337487) -> (continuation of Deferred@1759380579 after retry RPC after error@653820384) -> (continuation of Deferred@1243438092 after retry RPC after error@1631324885) -> (continuation of Deferred@2044323725 after retry RPC after error@458006231) -> (continuation of Deferred@990993550 after retry RPC after error@989496219) -> (continuation of Deferred@212463302 after retry RPC after error@1936483234) -> (continuation of Deferred@1118348397 after retry RPC after error@748683768) -> (continuation of Deferred@1874593140 after retry RPC after error@2136808018) -> (continuation of Deferred@2018595352 after retry RPC after error@1423057707) -> (continuation of Deferred@1531579232 after retry RPC after error@1280447953) -> (continuation of Deferred@1054953425 after retry RPC after error@2070935645) -> (continuation of Deferred@1895165275 after retry RPC after error@752194057) -> (continuation of Deferred@410963845 after retry RPC after error@483881415) -> (continuation of Deferred@1125636147 after retry RPC after error@1137856386) -> (continuation of Deferred@759083674 after retry RPC after error@180398003) -> (continuation of Deferred@2011898008 after retry RPC after error@638902200)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduTransaction.doCommitTransaction(KuduTransaction.java:491)
at org.apache.kudu.client.KuduTransaction.commitWithMode(KuduTransaction.java:537)
at org.apache.kudu.client.KuduTransaction.commit(KuduTransaction.java:284)
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveSwitchesToOtherTxnManager(TestKuduTransaction.java:1152)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1503)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1366)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1415)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.19.180.62:46081,127.19.180.61:37725,127.19.180.60:32797) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.62:46081(127.19.180.62:46081): Connection refused: /127.19.180.62:46081,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1503)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1366)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1415)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.180.61:37725(127.19.180.61:37725): Connection refused: /127.19.180.61:37725
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:326)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:342)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.19.180.61:37725
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)