07:01:06.026 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
07:01:06.026 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskjQXd4I/test-tmp
07:01:06.026 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/asan/bin
07:01:06.026 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskjQXd4I/java/build/dist-test/../../../build/asan/bin/kudu, test, mini_cluster, --serialization=pb]
07:01:06.028 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436"
extra_master_flags: "--txn_manager_enabled"
extra_master_flags: "--raft_heartbeat_interval_ms=100"
extra_tserver_flags: "--txn_keepalive_interval_ms=1000"
extra_tserver_flags: "--txn_staleness_tracker_interval_ms=250"
extra_tserver_flags: "--enable_txn_system_client_init=true"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
07:01:06.400 [DEBUG - main] (MiniKuduCluster.java:195) Response:
07:01:06.400 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
07:01:06.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:06.403015 9545 mini_oidc.cc:152] Starting JWKS server
07:01:06.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:06.406114 9545 webserver.cc:460] Webserver started at http://127.0.0.1:38595/ using document root <none> and password file <none>
07:01:06.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:06.406358 9545 mini_oidc.cc:189] Starting OIDC Discovery server
07:01:06.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:06.406934 9545 webserver.cc:460] Webserver started at http://127.0.0.1:44261/ using document root <none> and password file <none>
07:01:06.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T07:01:06Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
07:01:06.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T07:01:06Z Disabled control of system clock
07:01:06.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:06.440013 9545 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu
07:01:06.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu
07:01:06.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/master-0/wal
07:01:06.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/master-0/data
07:01:06.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:01:06.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:01:06.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:01:06.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:01:06.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:01:06.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:01:06.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/master-0/logs
07:01:06.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/master-0/data/info.pb
07:01:06.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:01:06.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:01:06.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:01:06.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:01:06.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:01:06.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:01:06.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:01:06.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:01:06.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:01:06.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:01:06.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:01:06.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:01:06.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.82.126:43671
07:01:06.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.82.126
07:01:06.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:01:06.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.82.84:34745
07:01:06.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:01:06.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:01:06.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:01:06.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:01:06.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.82.126:43671,127.9.82.125:45345,127.9.82.124:38119
07:01:06.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:01:06.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38595/jwks.json
07:01:06.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
07:01:06.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
07:01:06.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:06.768065 9556 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:01:06.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:06.768405 9556 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:01:06.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:06.768491 9556 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:01:06.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:06.775691 9556 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:01:06.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:06.775797 9556 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
07:01:06.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:06.775831 9556 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:01:06.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:06.775880 9556 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:01:06.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:06.775916 9556 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:01:06.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:06.775964 9556 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:01:06.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:06.776005 9556 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38595/jwks.json
07:01:06.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:06.784638 9556 master_runner.cc:386] Master server non-default flags:
07:01:06.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:01:06.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.82.84:34745
07:01:06.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:01:06.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:01:06.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
07:01:06.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/master-0/data
07:01:06.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/master-0/wal
07:01:06.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:01:06.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.82.126:43671,127.9.82.125:45345,127.9.82.124:38119
07:01:06.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
07:01:06.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:01:06.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:01:06.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:01:06.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.82.126:43671
07:01:06.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:01:06.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:01:06.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:01:06.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38595/jwks.json
07:01:06.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:01:06.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:01:06.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/master-0/data/info.pb
07:01:06.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.82.126
07:01:06.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:01:06.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:01:06.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:01:06.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:01:06.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:01:06.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:01:06.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/master-0/logs
07:01:06.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:01:06.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:01:06.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:01:06.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:01:06.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:01:06.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
07:01:06.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:01:06.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on efd1337a2aa3
07:01:06.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 251
07:01:06.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
07:01:06.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:06.786190 9556 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:01:06.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:06.788194 9556 file_cache.cc:492] Constructed file cache file cache with capacity 419430
07:01:06.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:06.799329 9561 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:01:06.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:06.799705 9562 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:01:06.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:06.800204 9564 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:01:06.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:06.800997 9556 server_base.cc:1034] running on GCE node
07:01:06.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:06.801784 9556 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:01:06.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:06.803366 9556 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:01:06.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:06.804575 9556 hybrid_clock.cc:648] HybridClock initialized: now 1714114866804518 us; error 48 us; skew 500 ppm
07:01:06.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:06.804934 9556 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:01:06.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:06.807346 9556 webserver.cc:460] Webserver started at http://127.9.82.126:44763/ using document root <none> and password file <none>
07:01:06.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:06.808163 9556 fs_manager.cc:359] Metadata directory not provided
07:01:06.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:06.808269 9556 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:01:06.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:06.808614 9556 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:01:06.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:06.811616 9556 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/master-0/data/instance:
07:01:06.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9089901da6614cedb5be9c0f69cef850"
07:01:06.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:01:06 on dist-test-slave-z59g"
07:01:06.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:06.812460 9556 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/master-0/wal/instance:
07:01:06.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9089901da6614cedb5be9c0f69cef850"
07:01:06.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:01:06 on dist-test-slave-z59g"
07:01:06.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:06.819475 9556 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.000s
07:01:06.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:06.823837 9570 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:01:06.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:06.825659 9556 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
07:01:06.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:06.825858 9556 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/master-0/data,/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/master-0/wal
07:01:06.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9089901da6614cedb5be9c0f69cef850"
07:01:06.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:01:06 on dist-test-slave-z59g"
07:01:06.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:06.826117 9556 fs_report.cc:383] FS layout report
07:01:06.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:01:06.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/master-0/wal
07:01:06.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/master-0/wal
07:01:06.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/master-0/data/data
07:01:06.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:01:06.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:01:06.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:01:06.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:01:06.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:01:06.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:01:06.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:01:06.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:01:06.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:01:06.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:01:06.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:01:06.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:06.842042 9556 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
07:01:06.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:06.843133 9556 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:01:06.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:06.843504 9556 kserver.cc:163] Server-wide thread pool size limit: 3276
07:01:06.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:06.866928 9631 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.9.82.126:43671 every 8 connection(s)
07:01:06.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:06.866956 9556 rpc_server.cc:307] RPC server started. Bound to: 127.9.82.126:43671
07:01:06.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:06.868681 9556 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/master-0/data/info.pb
07:01:06.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:06.873273 9632 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:01:06.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:06.876863 9545 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu as pid 9556
07:01:06.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:06.877172 9545 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/master-0/wal/instance
07:01:06.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:06.880165 9545 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu
07:01:06.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu
07:01:06.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/master-1/wal
07:01:06.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/master-1/data
07:01:06.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:01:06.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:01:06.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:01:06.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:01:06.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:01:06.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:01:06.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/master-1/logs
07:01:06.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/master-1/data/info.pb
07:01:06.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:01:06.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:01:06.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:01:06.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:01:06.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:01:06.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:01:06.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:01:06.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:01:06.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:01:06.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:01:06.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:01:06.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:01:06.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.82.125:45345
07:01:06.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.82.125
07:01:06.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:01:06.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.82.84:34745
07:01:06.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:01:06.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:01:06.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:01:06.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:01:06.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.82.126:43671,127.9.82.125:45345,127.9.82.124:38119
07:01:06.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:01:06.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38595/jwks.json
07:01:06.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
07:01:06.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
07:01:06.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:06.882198 9632 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.82.126" port: 43671 } has no permanent_uuid. Determining permanent_uuid...
07:01:06.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:06.898823 9632 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.82.125" port: 45345 } has no permanent_uuid. Determining permanent_uuid...
07:01:06.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:06.899811 9571 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.82.125:45345: connect: Connection refused (error 111)
07:01:06.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:06.901418 9632 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.9.82.125:45345: Network error: Client connection negotiation failed: client connection to 127.9.82.125:45345: connect: Connection refused (error 111)
07:01:06.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:06.950767 9632 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.82.125" port: 45345 } attempt: 1
07:01:06.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:06.952812 9632 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.9.82.125:45345: Network error: Client connection negotiation failed: client connection to 127.9.82.125:45345: connect: Connection refused (error 111)
07:01:07.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.021135 9632 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.82.125" port: 45345 } attempt: 2
07:01:07.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:07.023125 9632 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.9.82.125:45345: Network error: Client connection negotiation failed: client connection to 127.9.82.125:45345: connect: Connection refused (error 111)
07:01:07.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.114429 9632 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.82.125" port: 45345 } attempt: 3
07:01:07.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:07.116618 9632 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.9.82.125:45345: Network error: Client connection negotiation failed: client connection to 127.9.82.125:45345: connect: Connection refused (error 111)
07:01:07.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:07.226234 9634 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:01:07.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:07.226554 9634 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:01:07.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:07.226665 9634 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:01:07.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:07.233920 9634 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:01:07.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:07.234046 9634 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
07:01:07.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:07.234102 9634 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:01:07.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:07.234167 9634 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:01:07.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:07.234226 9634 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:01:07.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:07.234310 9634 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:01:07.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:07.234359 9634 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38595/jwks.json
07:01:07.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.242893 9634 master_runner.cc:386] Master server non-default flags:
07:01:07.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:01:07.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.82.84:34745
07:01:07.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:01:07.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:01:07.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
07:01:07.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/master-1/data
07:01:07.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/master-1/wal
07:01:07.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:01:07.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.82.126:43671,127.9.82.125:45345,127.9.82.124:38119
07:01:07.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
07:01:07.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:01:07.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:01:07.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:01:07.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.82.125:45345
07:01:07.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:01:07.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:01:07.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:01:07.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38595/jwks.json
07:01:07.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:01:07.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:01:07.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/master-1/data/info.pb
07:01:07.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.82.125
07:01:07.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:01:07.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:01:07.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:01:07.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:01:07.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:01:07.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:01:07.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/master-1/logs
07:01:07.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:01:07.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:01:07.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:01:07.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:01:07.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:01:07.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
07:01:07.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:01:07.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on efd1337a2aa3
07:01:07.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 251
07:01:07.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
07:01:07.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.244427 9634 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:01:07.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.246297 9634 file_cache.cc:492] Constructed file cache file cache with capacity 419430
07:01:07.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:07.258576 9642 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:01:07.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:07.259203 9643 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:01:07.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:07.259718 9645 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:01:07.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.259831 9632 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.82.125" port: 45345 } attempt: 4
07:01:07.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.260458 9634 server_base.cc:1034] running on GCE node
07:01:07.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.261226 9634 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:01:07.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:07.261965 9632 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.9.82.125:45345: Network error: Client connection negotiation failed: client connection to 127.9.82.125:45345: connect: Connection refused (error 111)
07:01:07.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.262682 9634 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:01:07.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.263865 9634 hybrid_clock.cc:648] HybridClock initialized: now 1714114867263824 us; error 42 us; skew 500 ppm
07:01:07.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.264144 9634 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:01:07.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.266197 9634 webserver.cc:460] Webserver started at http://127.9.82.125:34523/ using document root <none> and password file <none>
07:01:07.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.266916 9634 fs_manager.cc:359] Metadata directory not provided
07:01:07.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.266997 9634 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:01:07.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.267347 9634 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:01:07.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.269673 9634 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/master-1/data/instance:
07:01:07.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7b5c12662b074d3e903fbcd05ed4a533"
07:01:07.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:01:07 on dist-test-slave-z59g"
07:01:07.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.270524 9634 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/master-1/wal/instance:
07:01:07.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7b5c12662b074d3e903fbcd05ed4a533"
07:01:07.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:01:07 on dist-test-slave-z59g"
07:01:07.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.275048 9634 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.004s sys 0.000s
07:01:07.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.278017 9651 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:01:07.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.279196 9634 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
07:01:07.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.279379 9634 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/master-1/data,/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/master-1/wal
07:01:07.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7b5c12662b074d3e903fbcd05ed4a533"
07:01:07.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:01:07 on dist-test-slave-z59g"
07:01:07.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.279527 9634 fs_report.cc:383] FS layout report
07:01:07.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:01:07.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/master-1/wal
07:01:07.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/master-1/wal
07:01:07.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/master-1/data/data
07:01:07.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:01:07.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:01:07.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:01:07.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:01:07.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:01:07.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:01:07.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:01:07.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:01:07.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:01:07.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:01:07.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:01:07.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.303586 9634 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
07:01:07.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.304512 9634 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:01:07.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.304752 9634 kserver.cc:163] Server-wide thread pool size limit: 3276
07:01:07.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.326314 9712 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.9.82.125:45345 every 8 connection(s)
07:01:07.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.326339 9634 rpc_server.cc:307] RPC server started. Bound to: 127.9.82.125:45345
07:01:07.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.328080 9634 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/master-1/data/info.pb
07:01:07.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.329236 9545 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu as pid 9634
07:01:07.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.329460 9545 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/master-1/wal/instance
07:01:07.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.330818 9545 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu
07:01:07.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu
07:01:07.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/master-2/wal
07:01:07.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/master-2/data
07:01:07.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:01:07.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:01:07.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:01:07.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:01:07.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:01:07.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:01:07.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/master-2/logs
07:01:07.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/master-2/data/info.pb
07:01:07.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:01:07.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:01:07.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:01:07.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:01:07.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:01:07.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:01:07.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:01:07.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:01:07.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:01:07.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:01:07.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:01:07.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:01:07.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.82.124:38119
07:01:07.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.82.124
07:01:07.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:01:07.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.82.84:34745
07:01:07.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:01:07.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:01:07.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:01:07.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:01:07.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.82.126:43671,127.9.82.125:45345,127.9.82.124:38119
07:01:07.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:01:07.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38595/jwks.json
07:01:07.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
07:01:07.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
07:01:07.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.334000 9713 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:01:07.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.341615 9713 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.82.126" port: 43671 } has no permanent_uuid. Determining permanent_uuid...
07:01:07.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.357825 9713 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.82.125" port: 45345 } has no permanent_uuid. Determining permanent_uuid...
07:01:07.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.373804 9713 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.82.124" port: 38119 } has no permanent_uuid. Determining permanent_uuid...
07:01:07.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:07.375075 9654 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.82.124:38119: connect: Connection refused (error 111)
07:01:07.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:07.376842 9713 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.9.82.124:38119: Network error: Client connection negotiation failed: client connection to 127.9.82.124:38119: connect: Connection refused (error 111)
07:01:07.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.426178 9713 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.82.124" port: 38119 } attempt: 1
07:01:07.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:07.428275 9713 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.9.82.124:38119: Network error: Client connection negotiation failed: client connection to 127.9.82.124:38119: connect: Connection refused (error 111)
07:01:07.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.496507 9713 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.82.124" port: 38119 } attempt: 2
07:01:07.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:07.498358 9713 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.9.82.124:38119: Network error: Client connection negotiation failed: client connection to 127.9.82.124:38119: connect: Connection refused (error 111)
07:01:07.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.561280 9632 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.82.125" port: 45345 } attempt: 5
07:01:07.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.567847 9632 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.82.124" port: 38119 } has no permanent_uuid. Determining permanent_uuid...
07:01:07.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:07.569648 9632 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.9.82.124:38119: Network error: Client connection negotiation failed: client connection to 127.9.82.124:38119: connect: Connection refused (error 111)
07:01:07.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.589682 9713 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.82.124" port: 38119 } attempt: 3
07:01:07.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:07.591776 9713 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.9.82.124:38119: Network error: Client connection negotiation failed: client connection to 127.9.82.124:38119: connect: Connection refused (error 111)
07:01:07.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.620862 9632 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.82.124" port: 38119 } attempt: 1
07:01:07.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:07.622560 9632 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.9.82.124:38119: Network error: Client connection negotiation failed: client connection to 127.9.82.124:38119: connect: Connection refused (error 111)
07:01:07.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.690835 9632 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.82.124" port: 38119 } attempt: 2
07:01:07.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:07.692939 9632 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.9.82.124:38119: Network error: Client connection negotiation failed: client connection to 127.9.82.124:38119: connect: Connection refused (error 111)
07:01:07.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:07.697297 9715 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:01:07.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:07.697651 9715 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:01:07.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:07.697749 9715 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:01:07.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:07.704893 9715 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:01:07.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:07.705013 9715 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
07:01:07.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:07.705049 9715 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:01:07.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:07.705093 9715 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:01:07.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:07.705129 9715 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:01:07.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:07.705174 9715 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:01:07.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:07.705220 9715 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38595/jwks.json
07:01:07.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.714495 9715 master_runner.cc:386] Master server non-default flags:
07:01:07.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:01:07.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.82.84:34745
07:01:07.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:01:07.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:01:07.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
07:01:07.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/master-2/data
07:01:07.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/master-2/wal
07:01:07.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:01:07.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.82.126:43671,127.9.82.125:45345,127.9.82.124:38119
07:01:07.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
07:01:07.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:01:07.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:01:07.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:01:07.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.82.124:38119
07:01:07.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:01:07.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:01:07.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:01:07.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38595/jwks.json
07:01:07.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:01:07.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:01:07.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/master-2/data/info.pb
07:01:07.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.82.124
07:01:07.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:01:07.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:01:07.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:01:07.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:01:07.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:01:07.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:01:07.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/master-2/logs
07:01:07.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:01:07.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:01:07.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:01:07.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:01:07.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:01:07.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
07:01:07.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:01:07.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on efd1337a2aa3
07:01:07.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 251
07:01:07.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
07:01:07.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.716182 9715 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:01:07.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.718205 9715 file_cache.cc:492] Constructed file cache file cache with capacity 419430
07:01:07.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:07.730711 9726 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:01:07.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:07.731160 9723 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:01:07.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:07.731737 9724 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:01:07.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.732470 9715 server_base.cc:1034] running on GCE node
07:01:07.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.733353 9715 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:01:07.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.735119 9715 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:01:07.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.735028 9713 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.82.124" port: 38119 } attempt: 4
07:01:07.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.736342 9715 hybrid_clock.cc:648] HybridClock initialized: now 1714114867736266 us; error 72 us; skew 500 ppm
07:01:07.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.736794 9715 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:01:07.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:07.736832 9713 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.9.82.124:38119: Network error: Client connection negotiation failed: client connection to 127.9.82.124:38119: connect: Connection refused (error 111)
07:01:07.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.739213 9715 webserver.cc:460] Webserver started at http://127.9.82.124:37015/ using document root <none> and password file <none>
07:01:07.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.740171 9715 fs_manager.cc:359] Metadata directory not provided
07:01:07.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.740272 9715 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:01:07.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.740590 9715 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:01:07.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.743566 9715 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/master-2/data/instance:
07:01:07.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4a229e6d106c46caad88d051aa87c533"
07:01:07.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:01:07 on dist-test-slave-z59g"
07:01:07.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.744359 9715 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/master-2/wal/instance:
07:01:07.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4a229e6d106c46caad88d051aa87c533"
07:01:07.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:01:07 on dist-test-slave-z59g"
07:01:07.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.750738 9715 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.006s sys 0.000s
07:01:07.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.755227 9732 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:01:07.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.756914 9715 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
07:01:07.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.757102 9715 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/master-2/data,/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/master-2/wal
07:01:07.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4a229e6d106c46caad88d051aa87c533"
07:01:07.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:01:07 on dist-test-slave-z59g"
07:01:07.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.757328 9715 fs_report.cc:383] FS layout report
07:01:07.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:01:07.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/master-2/wal
07:01:07.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/master-2/wal
07:01:07.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/master-2/data/data
07:01:07.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:01:07.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:01:07.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:01:07.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:01:07.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:01:07.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:01:07.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:01:07.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:01:07.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:01:07.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:01:07.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:01:07.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.778334 9715 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
07:01:07.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.779541 9715 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:01:07.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.779875 9715 kserver.cc:163] Server-wide thread pool size limit: 3276
07:01:07.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.799170 9632 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.82.124" port: 38119 } attempt: 3
07:01:07.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.800882 9793 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.9.82.124:38119 every 8 connection(s)
07:01:07.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.800915 9715 rpc_server.cc:307] RPC server started. Bound to: 127.9.82.124:38119
07:01:07.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.802719 9715 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/master-2/data/info.pb
07:01:07.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.807191 9795 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:01:07.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.812538 9545 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu as pid 9715
07:01:07.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.812707 9545 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/master-2/wal/instance
07:01:07.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.814162 9795 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.82.126" port: 43671 } has no permanent_uuid. Determining permanent_uuid...
07:01:07.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.824457 9632 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9089901da6614cedb5be9c0f69cef850: Bootstrap starting.
07:01:07.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.830233 9632 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 9089901da6614cedb5be9c0f69cef850: Neither blocks nor log segments found. Creating new log.
07:01:07.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.832181 9632 log.cc:824] T 00000000000000000000000000000000 P 9089901da6614cedb5be9c0f69cef850: Log is configured to *not* fsync() on all Append() calls
07:01:07.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.836067 9632 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9089901da6614cedb5be9c0f69cef850: No bootstrap required, opened a new log
07:01:07.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.843204 9795 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.82.125" port: 45345 } has no permanent_uuid. Determining permanent_uuid...
07:01:07.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.845466 9632 raft_consensus.cc:348] T 00000000000000000000000000000000 P 9089901da6614cedb5be9c0f69cef850 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9089901da6614cedb5be9c0f69cef850" member_type: VOTER last_known_addr { host: "127.9.82.126" port: 43671 } } peers { permanent_uuid: "7b5c12662b074d3e903fbcd05ed4a533" member_type: VOTER last_known_addr { host: "127.9.82.125" port: 45345 } } peers { permanent_uuid: "4a229e6d106c46caad88d051aa87c533" member_type: VOTER last_known_addr { host: "127.9.82.124" port: 38119 } }
07:01:07.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.845922 9632 raft_consensus.cc:374] T 00000000000000000000000000000000 P 9089901da6614cedb5be9c0f69cef850 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:01:07.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.846112 9632 raft_consensus.cc:729] T 00000000000000000000000000000000 P 9089901da6614cedb5be9c0f69cef850 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9089901da6614cedb5be9c0f69cef850, State: Initialized, Role: FOLLOWER
07:01:07.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.847146 9632 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9089901da6614cedb5be9c0f69cef850 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9089901da6614cedb5be9c0f69cef850" member_type: VOTER last_known_addr { host: "127.9.82.126" port: 43671 } } peers { permanent_uuid: "7b5c12662b074d3e903fbcd05ed4a533" member_type: VOTER last_known_addr { host: "127.9.82.125" port: 45345 } } peers { permanent_uuid: "4a229e6d106c46caad88d051aa87c533" member_type: VOTER last_known_addr { host: "127.9.82.124" port: 38119 } }
07:01:07.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.848277 9802 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9089901da6614cedb5be9c0f69cef850 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9089901da6614cedb5be9c0f69cef850" member_type: VOTER last_known_addr { host: "127.9.82.126" port: 43671 } } peers { permanent_uuid: "7b5c12662b074d3e903fbcd05ed4a533" member_type: VOTER last_known_addr { host: "127.9.82.125" port: 45345 } } peers { permanent_uuid: "4a229e6d106c46caad88d051aa87c533" member_type: VOTER last_known_addr { host: "127.9.82.124" port: 38119 } } }
07:01:07.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.848642 9802 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9089901da6614cedb5be9c0f69cef850 [sys.catalog]: This master's current role is: FOLLOWER
07:01:07.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.850133 9632 sys_catalog.cc:564] T 00000000000000000000000000000000 P 9089901da6614cedb5be9c0f69cef850 [sys.catalog]: configured and running, proceeding with master startup.
07:01:07.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.864210 9795 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.82.124" port: 38119 } has no permanent_uuid. Determining permanent_uuid...
07:01:07.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.869138 9802 raft_consensus.cc:482] T 00000000000000000000000000000000 P 9089901da6614cedb5be9c0f69cef850 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:01:07.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.869405 9802 raft_consensus.cc:504] T 00000000000000000000000000000000 P 9089901da6614cedb5be9c0f69cef850 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9089901da6614cedb5be9c0f69cef850" member_type: VOTER last_known_addr { host: "127.9.82.126" port: 43671 } } peers { permanent_uuid: "7b5c12662b074d3e903fbcd05ed4a533" member_type: VOTER last_known_addr { host: "127.9.82.125" port: 45345 } } peers { permanent_uuid: "4a229e6d106c46caad88d051aa87c533" member_type: VOTER last_known_addr { host: "127.9.82.124" port: 38119 } }
07:01:07.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.872594 9802 leader_election.cc:288] T 00000000000000000000000000000000 P 9089901da6614cedb5be9c0f69cef850 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7b5c12662b074d3e903fbcd05ed4a533 (127.9.82.125:45345), 4a229e6d106c46caad88d051aa87c533 (127.9.82.124:38119)
07:01:07.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.877882 9795 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533: Bootstrap starting.
07:01:07.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:07.879442 9813 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 9089901da6614cedb5be9c0f69cef850: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:01:07.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:07.879559 9813 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:01:07.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.870337 9556 master_runner.cc:186] Error getting master registration for 127.9.82.125:45345: OK, instance_id { permanent_uuid: "7b5c12662b074d3e903fbcd05ed4a533" instance_seqno: 1714114867321095 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
07:01:07.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.880213 9556 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:01:07.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.882001 9795 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533: Neither blocks nor log segments found. Creating new log.
07:01:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.883260 9795 log.cc:824] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533: Log is configured to *not* fsync() on all Append() calls
07:01:07.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.887393 9795 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533: No bootstrap required, opened a new log
07:01:07.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.898561 9795 raft_consensus.cc:348] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9089901da6614cedb5be9c0f69cef850" member_type: VOTER last_known_addr { host: "127.9.82.126" port: 43671 } } peers { permanent_uuid: "7b5c12662b074d3e903fbcd05ed4a533" member_type: VOTER last_known_addr { host: "127.9.82.125" port: 45345 } } peers { permanent_uuid: "4a229e6d106c46caad88d051aa87c533" member_type: VOTER last_known_addr { host: "127.9.82.124" port: 38119 } }
07:01:07.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.898947 9795 raft_consensus.cc:374] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:01:07.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.899060 9795 raft_consensus.cc:729] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4a229e6d106c46caad88d051aa87c533, State: Initialized, Role: FOLLOWER
07:01:07.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.899861 9795 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9089901da6614cedb5be9c0f69cef850" member_type: VOTER last_known_addr { host: "127.9.82.126" port: 43671 } } peers { permanent_uuid: "7b5c12662b074d3e903fbcd05ed4a533" member_type: VOTER last_known_addr { host: "127.9.82.125" port: 45345 } } peers { permanent_uuid: "4a229e6d106c46caad88d051aa87c533" member_type: VOTER last_known_addr { host: "127.9.82.124" port: 38119 } }
07:01:07.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.902990 9816 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9089901da6614cedb5be9c0f69cef850" member_type: VOTER last_known_addr { host: "127.9.82.126" port: 43671 } } peers { permanent_uuid: "7b5c12662b074d3e903fbcd05ed4a533" member_type: VOTER last_known_addr { host: "127.9.82.125" port: 45345 } } peers { permanent_uuid: "4a229e6d106c46caad88d051aa87c533" member_type: VOTER last_known_addr { host: "127.9.82.124" port: 38119 } } }
07:01:07.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.903318 9816 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533 [sys.catalog]: This master's current role is: FOLLOWER
07:01:07.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.904314 9795 sys_catalog.cc:564] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533 [sys.catalog]: configured and running, proceeding with master startup.
07:01:07.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.904407 9759 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9089901da6614cedb5be9c0f69cef850" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4a229e6d106c46caad88d051aa87c533" is_pre_election: true
07:01:07.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.904964 9759 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9089901da6614cedb5be9c0f69cef850 in term 0.
07:01:07.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.905643 9573 leader_election.cc:302] T 00000000000000000000000000000000 P 9089901da6614cedb5be9c0f69cef850 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4a229e6d106c46caad88d051aa87c533, 9089901da6614cedb5be9c0f69cef850; no voters:
07:01:07.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.906165 9802 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 9089901da6614cedb5be9c0f69cef850 [term 0 FOLLOWER]: Leader pre-election won for term 1
07:01:07.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.906306 9802 raft_consensus.cc:482] T 00000000000000000000000000000000 P 9089901da6614cedb5be9c0f69cef850 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
07:01:07.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.906406 9802 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 9089901da6614cedb5be9c0f69cef850 [term 0 FOLLOWER]: Advancing to term 1
07:01:07.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.909572 9802 raft_consensus.cc:504] T 00000000000000000000000000000000 P 9089901da6614cedb5be9c0f69cef850 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9089901da6614cedb5be9c0f69cef850" member_type: VOTER last_known_addr { host: "127.9.82.126" port: 43671 } } peers { permanent_uuid: "7b5c12662b074d3e903fbcd05ed4a533" member_type: VOTER last_known_addr { host: "127.9.82.125" port: 45345 } } peers { permanent_uuid: "4a229e6d106c46caad88d051aa87c533" member_type: VOTER last_known_addr { host: "127.9.82.124" port: 38119 } }
07:01:07.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.912256 9759 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9089901da6614cedb5be9c0f69cef850" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4a229e6d106c46caad88d051aa87c533"
07:01:07.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.912467 9759 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533 [term 0 FOLLOWER]: Advancing to term 1
07:01:07.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.915642 9759 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9089901da6614cedb5be9c0f69cef850 in term 1.
07:01:07.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.916124 9573 leader_election.cc:302] T 00000000000000000000000000000000 P 9089901da6614cedb5be9c0f69cef850 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4a229e6d106c46caad88d051aa87c533, 9089901da6614cedb5be9c0f69cef850; no voters:
07:01:07.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.916558 9802 leader_election.cc:288] T 00000000000000000000000000000000 P 9089901da6614cedb5be9c0f69cef850 [CANDIDATE]: Term 1 election: Requested vote from peers 7b5c12662b074d3e903fbcd05ed4a533 (127.9.82.125:45345), 4a229e6d106c46caad88d051aa87c533 (127.9.82.124:38119)
07:01:07.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.916800 9802 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 9089901da6614cedb5be9c0f69cef850 [term 1 FOLLOWER]: Leader election won for term 1
07:01:07.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.916982 9802 raft_consensus.cc:686] T 00000000000000000000000000000000 P 9089901da6614cedb5be9c0f69cef850 [term 1 LEADER]: Becoming Leader. State: Replica: 9089901da6614cedb5be9c0f69cef850, State: Running, Role: LEADER
07:01:07.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.917634 9802 consensus_queue.cc:237] T 00000000000000000000000000000000 P 9089901da6614cedb5be9c0f69cef850 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9089901da6614cedb5be9c0f69cef850" member_type: VOTER last_known_addr { host: "127.9.82.126" port: 43671 } } peers { permanent_uuid: "7b5c12662b074d3e903fbcd05ed4a533" member_type: VOTER last_known_addr { host: "127.9.82.125" port: 45345 } } peers { permanent_uuid: "4a229e6d106c46caad88d051aa87c533" member_type: VOTER last_known_addr { host: "127.9.82.124" port: 38119 } }
07:01:07.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.926523 9823 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9089901da6614cedb5be9c0f69cef850 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9089901da6614cedb5be9c0f69cef850. Latest consensus state: current_term: 1 leader_uuid: "9089901da6614cedb5be9c0f69cef850" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9089901da6614cedb5be9c0f69cef850" member_type: VOTER last_known_addr { host: "127.9.82.126" port: 43671 } } peers { permanent_uuid: "7b5c12662b074d3e903fbcd05ed4a533" member_type: VOTER last_known_addr { host: "127.9.82.125" port: 45345 } } peers { permanent_uuid: "4a229e6d106c46caad88d051aa87c533" member_type: VOTER last_known_addr { host: "127.9.82.124" port: 38119 } } }
07:01:07.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.927178 9823 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9089901da6614cedb5be9c0f69cef850 [sys.catalog]: This master's current role is: LEADER
07:01:07.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.929705 9825 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
07:01:07.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.933049 9830 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
07:01:07.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.933950 9825 catalog_manager.cc:1484] Initializing Kudu cluster ID...
07:01:07.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.945094 9759 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533 [term 1 FOLLOWER]: Refusing update from remote peer 9089901da6614cedb5be9c0f69cef850: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
07:01:07.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.946627 9823 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 9089901da6614cedb5be9c0f69cef850 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4a229e6d106c46caad88d051aa87c533" member_type: VOTER last_known_addr { host: "127.9.82.124" port: 38119 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
07:01:07.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.946909 9715 master_runner.cc:186] Error getting master registration for 127.9.82.125:45345: OK, instance_id { permanent_uuid: "7b5c12662b074d3e903fbcd05ed4a533" instance_seqno: 1714114867321095 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
07:01:07.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.955937 9715 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:01:07.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.960038 9816 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9089901da6614cedb5be9c0f69cef850. Latest consensus state: current_term: 1 leader_uuid: "9089901da6614cedb5be9c0f69cef850" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9089901da6614cedb5be9c0f69cef850" member_type: VOTER last_known_addr { host: "127.9.82.126" port: 43671 } } peers { permanent_uuid: "7b5c12662b074d3e903fbcd05ed4a533" member_type: VOTER last_known_addr { host: "127.9.82.125" port: 45345 } } peers { permanent_uuid: "4a229e6d106c46caad88d051aa87c533" member_type: VOTER last_known_addr { host: "127.9.82.124" port: 38119 } } }
07:01:07.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.960328 9816 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533 [sys.catalog]: This master's current role is: FOLLOWER
07:01:07.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.962173 9823 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9089901da6614cedb5be9c0f69cef850 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "9089901da6614cedb5be9c0f69cef850" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9089901da6614cedb5be9c0f69cef850" member_type: VOTER last_known_addr { host: "127.9.82.126" port: 43671 } } peers { permanent_uuid: "7b5c12662b074d3e903fbcd05ed4a533" member_type: VOTER last_known_addr { host: "127.9.82.125" port: 45345 } } peers { permanent_uuid: "4a229e6d106c46caad88d051aa87c533" member_type: VOTER last_known_addr { host: "127.9.82.124" port: 38119 } } }
07:01:07.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.962689 9823 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9089901da6614cedb5be9c0f69cef850 [sys.catalog]: This master's current role is: LEADER
07:01:07.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.965553 9825 catalog_manager.cc:1347] Generated new cluster ID: 38c132b220504f5c90d8ff5a01d0e540
07:01:07.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.965972 9825 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
07:01:07.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.968545 9816 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "9089901da6614cedb5be9c0f69cef850" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9089901da6614cedb5be9c0f69cef850" member_type: VOTER last_known_addr { host: "127.9.82.126" port: 43671 } } peers { permanent_uuid: "7b5c12662b074d3e903fbcd05ed4a533" member_type: VOTER last_known_addr { host: "127.9.82.125" port: 45345 } } peers { permanent_uuid: "4a229e6d106c46caad88d051aa87c533" member_type: VOTER last_known_addr { host: "127.9.82.124" port: 38119 } } }
07:01:07.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.968744 9816 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533 [sys.catalog]: This master's current role is: FOLLOWER
07:01:07.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.981263 9825 catalog_manager.cc:1370] Generated new certificate authority record
07:01:07.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:07.983382 9825 catalog_manager.cc:1504] Loading token signing keys...
07:01:08.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.001505 9825 catalog_manager.cc:5857] T 00000000000000000000000000000000 P 9089901da6614cedb5be9c0f69cef850: Generated new TSK 0
07:01:08.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.003167 9825 catalog_manager.cc:1514] Initializing in-progress tserver states...
07:01:08.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.024627 9545 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu
07:01:08.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu
07:01:08.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/ts-0/wal
07:01:08.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/ts-0/data
07:01:08.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:01:08.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:01:08.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:01:08.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:01:08.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:01:08.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:01:08.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/ts-0/logs
07:01:08.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/ts-0/data/info.pb
07:01:08.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:01:08.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:01:08.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:01:08.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:01:08.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:01:08.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:01:08.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:01:08.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:01:08.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
07:01:08.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:01:08.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.82.65:0
07:01:08.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.82.65
07:01:08.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.82.65
07:01:08.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:01:08.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.82.126:43671,127.9.82.125:45345,127.9.82.124:38119
07:01:08.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.82.84:34745
07:01:08.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:01:08.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:01:08.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:01:08.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
07:01:08.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
07:01:08.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
07:01:08.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.036027 9713 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.82.124" port: 38119 } attempt: 5
07:01:08.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.049721 9713 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533: Bootstrap starting.
07:01:08.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.055009 9713 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533: Neither blocks nor log segments found. Creating new log.
07:01:08.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.056960 9713 log.cc:824] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533: Log is configured to *not* fsync() on all Append() calls
07:01:08.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.069859 9713 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533: No bootstrap required, opened a new log
07:01:08.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.084715 9713 raft_consensus.cc:348] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9089901da6614cedb5be9c0f69cef850" member_type: VOTER last_known_addr { host: "127.9.82.126" port: 43671 } } peers { permanent_uuid: "7b5c12662b074d3e903fbcd05ed4a533" member_type: VOTER last_known_addr { host: "127.9.82.125" port: 45345 } } peers { permanent_uuid: "4a229e6d106c46caad88d051aa87c533" member_type: VOTER last_known_addr { host: "127.9.82.124" port: 38119 } }
07:01:08.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.085588 9713 raft_consensus.cc:374] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:01:08.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.086038 9713 raft_consensus.cc:729] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7b5c12662b074d3e903fbcd05ed4a533, State: Initialized, Role: FOLLOWER
07:01:08.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.087597 9713 consensus_queue.cc:260] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9089901da6614cedb5be9c0f69cef850" member_type: VOTER last_known_addr { host: "127.9.82.126" port: 43671 } } peers { permanent_uuid: "7b5c12662b074d3e903fbcd05ed4a533" member_type: VOTER last_known_addr { host: "127.9.82.125" port: 45345 } } peers { permanent_uuid: "4a229e6d106c46caad88d051aa87c533" member_type: VOTER last_known_addr { host: "127.9.82.124" port: 38119 } }
07:01:08.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.089434 9838 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9089901da6614cedb5be9c0f69cef850" member_type: VOTER last_known_addr { host: "127.9.82.126" port: 43671 } } peers { permanent_uuid: "7b5c12662b074d3e903fbcd05ed4a533" member_type: VOTER last_known_addr { host: "127.9.82.125" port: 45345 } } peers { permanent_uuid: "4a229e6d106c46caad88d051aa87c533" member_type: VOTER last_known_addr { host: "127.9.82.124" port: 38119 } } }
07:01:08.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.090104 9838 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [sys.catalog]: This master's current role is: FOLLOWER
07:01:08.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.091375 9677 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9089901da6614cedb5be9c0f69cef850" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7b5c12662b074d3e903fbcd05ed4a533"
07:01:08.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.091964 9676 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 0 FOLLOWER]: Advancing to term 1
07:01:08.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.093235 9678 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9089901da6614cedb5be9c0f69cef850" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7b5c12662b074d3e903fbcd05ed4a533" is_pre_election: true
07:01:08.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.095345 9713 sys_catalog.cc:564] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [sys.catalog]: configured and running, proceeding with master startup.
07:01:08.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.096082 9676 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 1 FOLLOWER]: Refusing update from remote peer 9089901da6614cedb5be9c0f69cef850: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
07:01:08.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.097182 9823 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 9089901da6614cedb5be9c0f69cef850 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b5c12662b074d3e903fbcd05ed4a533" member_type: VOTER last_known_addr { host: "127.9.82.125" port: 45345 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
07:01:08.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.097280 9838 raft_consensus.cc:482] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:01:08.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.097481 9838 raft_consensus.cc:504] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9089901da6614cedb5be9c0f69cef850" member_type: VOTER last_known_addr { host: "127.9.82.126" port: 43671 } } peers { permanent_uuid: "7b5c12662b074d3e903fbcd05ed4a533" member_type: VOTER last_known_addr { host: "127.9.82.125" port: 45345 } } peers { permanent_uuid: "4a229e6d106c46caad88d051aa87c533" member_type: VOTER last_known_addr { host: "127.9.82.124" port: 38119 } }
07:01:08.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.100206 9838 leader_election.cc:288] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 9089901da6614cedb5be9c0f69cef850 (127.9.82.126:43671), 4a229e6d106c46caad88d051aa87c533 (127.9.82.124:38119)
07:01:08.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.101051 9597 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7b5c12662b074d3e903fbcd05ed4a533" candidate_term: 2 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9089901da6614cedb5be9c0f69cef850" is_pre_election: true
07:01:08.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.102102 9759 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7b5c12662b074d3e903fbcd05ed4a533" candidate_term: 2 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4a229e6d106c46caad88d051aa87c533" is_pre_election: true
07:01:08.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.103281 9655 leader_election.cc:302] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 7b5c12662b074d3e903fbcd05ed4a533; no voters: 4a229e6d106c46caad88d051aa87c533, 9089901da6614cedb5be9c0f69cef850
07:01:08.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.146000 9840 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
07:01:08.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.146211 9838 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9089901da6614cedb5be9c0f69cef850. Latest consensus state: current_term: 1 leader_uuid: "9089901da6614cedb5be9c0f69cef850" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9089901da6614cedb5be9c0f69cef850" member_type: VOTER last_known_addr { host: "127.9.82.126" port: 43671 } } peers { permanent_uuid: "7b5c12662b074d3e903fbcd05ed4a533" member_type: VOTER last_known_addr { host: "127.9.82.125" port: 45345 } } peers { permanent_uuid: "4a229e6d106c46caad88d051aa87c533" member_type: VOTER last_known_addr { host: "127.9.82.124" port: 38119 } } }
07:01:08.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.146497 9838 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [sys.catalog]: This master's current role is: FOLLOWER
07:01:08.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.146839 9845 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "9089901da6614cedb5be9c0f69cef850" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9089901da6614cedb5be9c0f69cef850" member_type: VOTER last_known_addr { host: "127.9.82.126" port: 43671 } } peers { permanent_uuid: "7b5c12662b074d3e903fbcd05ed4a533" member_type: VOTER last_known_addr { host: "127.9.82.125" port: 45345 } } peers { permanent_uuid: "4a229e6d106c46caad88d051aa87c533" member_type: VOTER last_known_addr { host: "127.9.82.124" port: 38119 } } }
07:01:08.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.147351 9845 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [sys.catalog]: This master's current role is: FOLLOWER
07:01:08.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.174309 9855 catalog_manager.cc:1259] Loaded cluster ID: 38c132b220504f5c90d8ff5a01d0e540
07:01:08.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.174647 9855 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533: loading cluster ID for follower catalog manager: success
07:01:08.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.178279 9855 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533: acquiring CA information for follower catalog manager: success
07:01:08.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.180981 9855 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
07:01:08.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.194545 9823 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9089901da6614cedb5be9c0f69cef850 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "9089901da6614cedb5be9c0f69cef850" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9089901da6614cedb5be9c0f69cef850" member_type: VOTER last_known_addr { host: "127.9.82.126" port: 43671 } } peers { permanent_uuid: "7b5c12662b074d3e903fbcd05ed4a533" member_type: VOTER last_known_addr { host: "127.9.82.125" port: 45345 } } peers { permanent_uuid: "4a229e6d106c46caad88d051aa87c533" member_type: VOTER last_known_addr { host: "127.9.82.124" port: 38119 } } }
07:01:08.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.194860 9823 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9089901da6614cedb5be9c0f69cef850 [sys.catalog]: This master's current role is: LEADER
07:01:08.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:08.441771 9836 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:01:08.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:08.442082 9836 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:01:08.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:08.442239 9836 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:01:08.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:08.449461 9836 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:01:08.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:08.449645 9836 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
07:01:08.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:08.449697 9836 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
07:01:08.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:08.449759 9836 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
07:01:08.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:08.449855 9836 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.9.82.65
07:01:08.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.458590 9836 tablet_server_runner.cc:78] Tablet server non-default flags:
07:01:08.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:01:08.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.82.84:34745
07:01:08.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:01:08.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:01:08.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/ts-0/data
07:01:08.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/ts-0/wal
07:01:08.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:01:08.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:01:08.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.82.65:0
07:01:08.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:01:08.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:01:08.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:01:08.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/ts-0/data/info.pb
07:01:08.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.82.65
07:01:08.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:01:08.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
07:01:08.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
07:01:08.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
07:01:08.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.82.126:43671,127.9.82.125:45345,127.9.82.124:38119
07:01:08.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:01:08.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:01:08.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:01:08.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:01:08.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:01:08.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.82.65
07:01:08.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/ts-0/logs
07:01:08.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:01:08.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:01:08.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:01:08.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
07:01:08.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:01:08.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
07:01:08.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:01:08.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on efd1337a2aa3
07:01:08.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 251
07:01:08.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
07:01:08.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.460263 9836 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:01:08.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.462357 9836 file_cache.cc:492] Constructed file cache file cache with capacity 419430
07:01:08.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:08.475522 9864 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:01:08.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:08.476130 9861 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:01:08.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:08.476619 9862 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:01:08.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.477259 9836 server_base.cc:1034] running on GCE node
07:01:08.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.477756 9836 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:01:08.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.478636 9836 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:01:08.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.479833 9836 hybrid_clock.cc:648] HybridClock initialized: now 1714114868479779 us; error 48 us; skew 500 ppm
07:01:08.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.480177 9836 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:01:08.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.482611 9836 webserver.cc:460] Webserver started at http://127.9.82.65:34173/ using document root <none> and password file <none>
07:01:08.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.483356 9836 fs_manager.cc:359] Metadata directory not provided
07:01:08.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.483465 9836 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:01:08.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.483737 9836 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:01:08.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.486150 9836 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/ts-0/data/instance:
07:01:08.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "11ae612cdfd94d638c0c35046ab8118d"
07:01:08.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:01:08 on dist-test-slave-z59g"
07:01:08.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.486900 9836 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/ts-0/wal/instance:
07:01:08.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "11ae612cdfd94d638c0c35046ab8118d"
07:01:08.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:01:08 on dist-test-slave-z59g"
07:01:08.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.492368 9836 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.008s sys 0.000s
07:01:08.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.495636 9870 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:01:08.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.497006 9836 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.001s
07:01:08.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.497133 9836 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/ts-0/data,/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/ts-0/wal
07:01:08.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "11ae612cdfd94d638c0c35046ab8118d"
07:01:08.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:01:08 on dist-test-slave-z59g"
07:01:08.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.497287 9836 fs_report.cc:383] FS layout report
07:01:08.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:01:08.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/ts-0/wal
07:01:08.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/ts-0/wal
07:01:08.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/ts-0/data/data
07:01:08.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:01:08.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:01:08.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:01:08.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:01:08.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:01:08.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:01:08.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:01:08.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:01:08.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:01:08.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:01:08.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:01:08.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.525601 9836 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
07:01:08.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.526548 9836 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:01:08.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.526793 9836 kserver.cc:163] Server-wide thread pool size limit: 3276
07:01:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.530033 9836 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
07:01:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.530119 9836 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
07:01:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.530273 9836 ts_tablet_manager.cc:590] Registered 0 tablets
07:01:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.530449 9836 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.001s sys 0.000s
07:01:08.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.620808 9836 rpc_server.cc:307] RPC server started. Bound to: 127.9.82.65:46241
07:01:08.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.620882 9994 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.9.82.65:46241 every 8 connection(s)
07:01:08.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.622273 9836 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/ts-0/data/info.pb
07:01:08.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.624555 9996 heartbeater.cc:347] Connected to a master server at 127.9.82.126:43671
07:01:08.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.624955 9996 heartbeater.cc:464] Registering TS with master...
07:01:08.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.626155 9996 heartbeater.cc:511] Master 127.9.82.126:43671 requested a full tablet report, sending...
07:01:08.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.628726 9587 ts_manager.cc:194] Registered new tserver with Master: 11ae612cdfd94d638c0c35046ab8118d (127.9.82.65:46241)
07:01:08.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.631132 9545 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu as pid 9836
07:01:08.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.631564 9545 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/ts-0/wal/instance
07:01:08.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.635824 9545 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu
07:01:08.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu
07:01:08.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/ts-1/wal
07:01:08.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/ts-1/data
07:01:08.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:01:08.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:01:08.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:01:08.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:01:08.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:01:08.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:01:08.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/ts-1/logs
07:01:08.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/ts-1/data/info.pb
07:01:08.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:01:08.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:01:08.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:01:08.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:01:08.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:01:08.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:01:08.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:01:08.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:01:08.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
07:01:08.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:01:08.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.82.66:0
07:01:08.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.82.66
07:01:08.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.82.66
07:01:08.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:01:08.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.82.126:43671,127.9.82.125:45345,127.9.82.124:38119
07:01:08.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.82.84:34745
07:01:08.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:01:08.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:01:08.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:01:08.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
07:01:08.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
07:01:08.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
07:01:08.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.637379 9587 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.9.82.65:50801
07:01:08.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.637611 9995 heartbeater.cc:347] Connected to a master server at 127.9.82.124:38119
07:01:08.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.637794 9995 heartbeater.cc:464] Registering TS with master...
07:01:08.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.638459 9995 heartbeater.cc:511] Master 127.9.82.124:38119 requested a full tablet report, sending...
07:01:08.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.641008 9749 ts_manager.cc:194] Registered new tserver with Master: 11ae612cdfd94d638c0c35046ab8118d (127.9.82.65:46241)
07:01:08.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.653009 9997 heartbeater.cc:347] Connected to a master server at 127.9.82.125:45345
07:01:08.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.653193 9997 heartbeater.cc:464] Registering TS with master...
07:01:08.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.653584 9997 heartbeater.cc:511] Master 127.9.82.125:45345 requested a full tablet report, sending...
07:01:08.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.655514 9667 ts_manager.cc:194] Registered new tserver with Master: 11ae612cdfd94d638c0c35046ab8118d (127.9.82.65:46241)
07:01:08.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.938254 9830 catalog_manager.cc:1259] Loaded cluster ID: 38c132b220504f5c90d8ff5a01d0e540
07:01:08.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.938601 9830 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533: loading cluster ID for follower catalog manager: success
07:01:08.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.942559 9830 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533: acquiring CA information for follower catalog manager: success
07:01:08.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:08.945035 9830 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
07:01:09.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:09.007092 9998 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:01:09.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:09.007668 9998 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:01:09.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:09.007863 9998 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:01:09.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:09.016129 9998 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:01:09.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:09.016317 9998 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
07:01:09.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:09.016357 9998 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
07:01:09.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:09.016404 9998 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
07:01:09.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:09.016472 9998 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.9.82.66
07:01:09.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.025887 9998 tablet_server_runner.cc:78] Tablet server non-default flags:
07:01:09.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:01:09.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.82.84:34745
07:01:09.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:01:09.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:01:09.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/ts-1/data
07:01:09.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/ts-1/wal
07:01:09.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:01:09.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:01:09.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.82.66:0
07:01:09.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:01:09.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:01:09.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:01:09.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/ts-1/data/info.pb
07:01:09.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.82.66
07:01:09.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:01:09.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
07:01:09.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
07:01:09.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
07:01:09.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.82.126:43671,127.9.82.125:45345,127.9.82.124:38119
07:01:09.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:01:09.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:01:09.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:01:09.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:01:09.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:01:09.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.82.66
07:01:09.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/ts-1/logs
07:01:09.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:01:09.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:01:09.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:01:09.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
07:01:09.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:01:09.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
07:01:09.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:01:09.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on efd1337a2aa3
07:01:09.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 251
07:01:09.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
07:01:09.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.027690 9998 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:01:09.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.029748 9998 file_cache.cc:492] Constructed file cache file cache with capacity 419430
07:01:09.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:09.043670 10005 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:01:09.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:09.044188 10006 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:01:09.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:09.044796 10008 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:01:09.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.049194 9998 server_base.cc:1034] running on GCE node
07:01:09.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.049861 9998 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:01:09.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.051462 9998 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:01:09.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.052693 9998 hybrid_clock.cc:648] HybridClock initialized: now 1714114869052617 us; error 70 us; skew 500 ppm
07:01:09.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.053035 9998 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:01:09.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.055470 9998 webserver.cc:460] Webserver started at http://127.9.82.66:41751/ using document root <none> and password file <none>
07:01:09.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.056388 9998 fs_manager.cc:359] Metadata directory not provided
07:01:09.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.056490 9998 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:01:09.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.056804 9998 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:01:09.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.059419 9998 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/ts-1/data/instance:
07:01:09.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5d3db908b485446cbcbe6827b7bbbcd7"
07:01:09.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:01:09 on dist-test-slave-z59g"
07:01:09.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.060231 9998 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/ts-1/wal/instance:
07:01:09.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5d3db908b485446cbcbe6827b7bbbcd7"
07:01:09.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:01:09 on dist-test-slave-z59g"
07:01:09.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.065657 9998 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.005s sys 0.000s
07:01:09.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.069187 10014 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:01:09.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.070559 9998 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
07:01:09.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.070741 9998 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/ts-1/data,/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/ts-1/wal
07:01:09.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5d3db908b485446cbcbe6827b7bbbcd7"
07:01:09.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:01:09 on dist-test-slave-z59g"
07:01:09.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.070974 9998 fs_report.cc:383] FS layout report
07:01:09.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:01:09.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/ts-1/wal
07:01:09.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/ts-1/wal
07:01:09.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/ts-1/data/data
07:01:09.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:01:09.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:01:09.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:01:09.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:01:09.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:01:09.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:01:09.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:01:09.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:01:09.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:01:09.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:01:09.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:01:09.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.086351 9998 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
07:01:09.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.087813 9998 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:01:09.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.088138 9998 kserver.cc:163] Server-wide thread pool size limit: 3276
07:01:09.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.091601 9998 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
07:01:09.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.091703 9998 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
07:01:09.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.091827 9998 ts_tablet_manager.cc:590] Registered 0 tablets
07:01:09.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.091883 9998 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
07:01:09.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.181183 9998 rpc_server.cc:307] RPC server started. Bound to: 127.9.82.66:37375
07:01:09.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.181200 10136 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.9.82.66:37375 every 8 connection(s)
07:01:09.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.182710 9998 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/ts-1/data/info.pb
07:01:09.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.184033 9545 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu as pid 9998
07:01:09.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.184219 9545 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/ts-1/wal/instance
07:01:09.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.185618 10138 heartbeater.cc:347] Connected to a master server at 127.9.82.126:43671
07:01:09.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.186199 10138 heartbeater.cc:464] Registering TS with master...
07:01:09.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.187888 10138 heartbeater.cc:511] Master 127.9.82.126:43671 requested a full tablet report, sending...
07:01:09.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.190436 9587 ts_manager.cc:194] Registered new tserver with Master: 5d3db908b485446cbcbe6827b7bbbcd7 (127.9.82.66:37375)
07:01:09.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.191697 9587 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.9.82.66:55131
07:01:09.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.194468 9545 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu
07:01:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu
07:01:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/ts-2/wal
07:01:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/ts-2/data
07:01:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:01:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:01:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:01:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:01:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:01:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:01:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/ts-2/logs
07:01:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/ts-2/data/info.pb
07:01:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:01:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:01:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:01:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:01:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:01:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:01:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:01:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:01:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
07:01:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:01:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.82.67:0
07:01:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.82.67
07:01:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.82.67
07:01:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:01:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.82.126:43671,127.9.82.125:45345,127.9.82.124:38119
07:01:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.82.84:34745
07:01:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:01:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:01:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:01:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
07:01:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
07:01:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
07:01:09.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.199970 10139 heartbeater.cc:347] Connected to a master server at 127.9.82.125:45345
07:01:09.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.200150 10139 heartbeater.cc:464] Registering TS with master...
07:01:09.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.200732 10139 heartbeater.cc:511] Master 127.9.82.125:45345 requested a full tablet report, sending...
07:01:09.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.201884 9667 ts_manager.cc:194] Registered new tserver with Master: 5d3db908b485446cbcbe6827b7bbbcd7 (127.9.82.66:37375)
07:01:09.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.212822 10137 heartbeater.cc:347] Connected to a master server at 127.9.82.124:38119
07:01:09.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.213268 10137 heartbeater.cc:464] Registering TS with master...
07:01:09.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.213717 10137 heartbeater.cc:511] Master 127.9.82.124:38119 requested a full tablet report, sending...
07:01:09.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.215169 9749 ts_manager.cc:194] Registered new tserver with Master: 5d3db908b485446cbcbe6827b7bbbcd7 (127.9.82.66:37375)
07:01:09.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:09.569305 10141 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:01:09.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:09.569554 10141 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:01:09.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:09.569653 10141 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:01:09.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:09.576251 10141 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:01:09.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:09.576372 10141 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
07:01:09.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:09.576412 10141 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
07:01:09.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:09.576460 10141 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
07:01:09.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:09.576524 10141 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.9.82.67
07:01:09.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.584661 10141 tablet_server_runner.cc:78] Tablet server non-default flags:
07:01:09.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:01:09.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.82.84:34745
07:01:09.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:01:09.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:01:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/ts-2/data
07:01:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/ts-2/wal
07:01:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:01:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:01:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.82.67:0
07:01:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:01:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:01:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:01:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/ts-2/data/info.pb
07:01:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.82.67
07:01:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:01:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
07:01:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
07:01:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
07:01:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.82.126:43671,127.9.82.125:45345,127.9.82.124:38119
07:01:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:01:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:01:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:01:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:01:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:01:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.82.67
07:01:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/ts-2/logs
07:01:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:01:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:01:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:01:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
07:01:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:01:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
07:01:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:01:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on efd1337a2aa3
07:01:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 251
07:01:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
07:01:09.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.586074 10141 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:01:09.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.588001 10141 file_cache.cc:492] Constructed file cache file cache with capacity 419430
07:01:09.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:09.601051 10146 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:01:09.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:09.601596 10149 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:01:09.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:09.602118 10147 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:01:09.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.602739 10141 server_base.cc:1034] running on GCE node
07:01:09.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.603365 10141 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:01:09.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.604419 10141 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:01:09.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.605602 10141 hybrid_clock.cc:648] HybridClock initialized: now 1714114869605538 us; error 72 us; skew 500 ppm
07:01:09.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.605908 10141 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:01:09.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.611925 10141 webserver.cc:460] Webserver started at http://127.9.82.67:44721/ using document root <none> and password file <none>
07:01:09.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.612876 10141 fs_manager.cc:359] Metadata directory not provided
07:01:09.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.612968 10141 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:01:09.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.613235 10141 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:01:09.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.616096 10141 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/ts-2/data/instance:
07:01:09.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5289e655d9c647f499e5df5f236ce008"
07:01:09.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:01:09 on dist-test-slave-z59g"
07:01:09.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.616816 10141 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/ts-2/wal/instance:
07:01:09.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5289e655d9c647f499e5df5f236ce008"
07:01:09.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:01:09 on dist-test-slave-z59g"
07:01:09.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.622573 10141 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.006s sys 0.000s
07:01:09.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.626295 10155 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:01:09.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.627822 10141 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
07:01:09.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.627985 10141 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/ts-2/data,/tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/ts-2/wal
07:01:09.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5289e655d9c647f499e5df5f236ce008"
07:01:09.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:01:09 on dist-test-slave-z59g"
07:01:09.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.628176 10141 fs_report.cc:383] FS layout report
07:01:09.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:01:09.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/ts-2/wal
07:01:09.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/ts-2/wal
07:01:09.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/ts-2/data/data
07:01:09.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:01:09.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:01:09.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:01:09.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:01:09.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:01:09.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:01:09.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:01:09.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:01:09.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:01:09.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:01:09.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:01:09.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.645131 9996 heartbeater.cc:503] Master 127.9.82.126:43671 was elected leader, sending a full tablet report...
07:01:09.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.649220 10141 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
07:01:09.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.650135 10141 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:01:09.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.650460 10141 kserver.cc:163] Server-wide thread pool size limit: 3276
07:01:09.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.653812 10141 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
07:01:09.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.653908 10141 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
07:01:09.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.654035 10141 ts_tablet_manager.cc:590] Registered 0 tablets
07:01:09.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.654273 10141 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.001s sys 0.000s
07:01:09.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.702522 10141 rpc_server.cc:307] RPC server started. Bound to: 127.9.82.67:36991
07:01:09.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.702551 10277 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.9.82.67:36991 every 8 connection(s)
07:01:09.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.703949 10141 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/ts-2/data/info.pb
07:01:09.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.706697 10279 heartbeater.cc:347] Connected to a master server at 127.9.82.126:43671
07:01:09.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.707796 10279 heartbeater.cc:464] Registering TS with master...
07:01:09.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.708865 10279 heartbeater.cc:511] Master 127.9.82.126:43671 requested a full tablet report, sending...
07:01:09.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.710656 9587 ts_manager.cc:194] Registered new tserver with Master: 5289e655d9c647f499e5df5f236ce008 (127.9.82.67:36991)
07:01:09.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.711824 9587 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.9.82.67:45681
07:01:09.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.712502 9545 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu as pid 10141
07:01:09.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.712677 9545 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskjQXd4I/test-tmp/mini-kudu-cluster7285652116495837436/ts-2/wal/instance
07:01:09.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.714005 10278 heartbeater.cc:347] Connected to a master server at 127.9.82.124:38119
07:01:09.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.714303 10278 heartbeater.cc:464] Registering TS with master...
07:01:09.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.714592 10280 heartbeater.cc:347] Connected to a master server at 127.9.82.125:45345
07:01:09.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.715096 10280 heartbeater.cc:464] Registering TS with master...
07:01:09.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.715296 10278 heartbeater.cc:511] Master 127.9.82.124:38119 requested a full tablet report, sending...
07:01:09.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.715540 10280 heartbeater.cc:511] Master 127.9.82.125:45345 requested a full tablet report, sending...
07:01:09.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.716418 9749 ts_manager.cc:194] Registered new tserver with Master: 5289e655d9c647f499e5df5f236ce008 (127.9.82.67:36991)
07:01:09.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.716622 9668 ts_manager.cc:194] Registered new tserver with Master: 5289e655d9c647f499e5df5f236ce008 (127.9.82.67:36991)
07:01:09.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.720733 9545 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
07:01:09.722 [DEBUG - main] (MiniKuduCluster.java:195) Response:
07:01:09.722 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
07:01:09.723 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.9.82.126"
port: 43671
}
bound_http_address {
host: "127.9.82.126"
port: 44763
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.9.82.125"
port: 45345
}
bound_http_address {
host: "127.9.82.125"
port: 34523
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.9.82.124"
port: 38119
}
bound_http_address {
host: "127.9.82.124"
port: 37015
}
}
}
07:01:09.723 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
07:01:09.724 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.9.82.65"
port: 46241
}
bound_http_address {
host: "127.9.82.65"
port: 34173
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.9.82.66"
port: 37375
}
bound_http_address {
host: "127.9.82.66"
port: 41751
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.9.82.67"
port: 36991
}
bound_http_address {
host: "127.9.82.67"
port: 44721
}
}
}
07:01:09.724 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
07:01:09.725 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
07:01:09.727 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.9.82.126:43671(127.9.82.126:43671)] connecting to peer
07:01:09.728 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.9.82.124:38119(127.9.82.124:38119)] connecting to peer
07:01:09.728 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.9.82.126:43671(127.9.82.126:43671)] Successfully connected to peer
07:01:09.729 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.9.82.125:45345(127.9.82.125:45345)] connecting to peer
07:01:09.730 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.9.82.126
07:01:09.730 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.9.82.124:38119(127.9.82.124:38119)] Successfully connected to peer
07:01:09.732 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.9.82.124
07:01:09.735 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.9.82.125:45345(127.9.82.125:45345)] Successfully connected to peer
07:01:09.742 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.9.82.125
07:01:09.744 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xc2b30328, L:/127.0.0.1:38182 - R:/127.9.82.124:38119] using SASL/PLAIN
07:01:09.745 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xf26d7e63, L:/127.0.0.1:35410 - R:/127.9.82.126:43671] using SASL/PLAIN
07:01:09.752 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
07:01:09.752 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=26), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [21ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: OK, [25ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: OK, deferred=null) after lookup
07:01:09.753 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xaceb8bfb, L:/127.0.0.1:50324 - R:/127.9.82.125:45345] using SASL/PLAIN
07:01:09.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.754621 9587 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:35410:
07:01:09.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_manager_fallback"
07:01:09.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
07:01:09.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:01:09.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
07:01:09.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
07:01:09.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
07:01:09.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:01:09.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:01:09.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:01:09.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:01:09.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:01:09.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:01:09.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column1_i"
07:01:09.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
07:01:09.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:01:09.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:01:09.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:01:09.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:01:09.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:01:09.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:01:09.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:01:09.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column2_i"
07:01:09.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
07:01:09.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:01:09.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:01:09.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:01:09.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:01:09.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:01:09.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:01:09.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:01:09.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column3_s"
07:01:09.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
07:01:09.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:01:09.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
07:01:09.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: DICT_ENCODING
07:01:09.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: LZ4
07:01:09.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 4096
07:01:09.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:01:09.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:01:09.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:01:09.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:01:09.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column4_b"
07:01:09.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: BOOL
07:01:09.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:01:09.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:01:09.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:01:09.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:01:09.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
07:01:09.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:01:09.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:01:09.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
07:01:09.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) hash_schema {
07:01:09.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:01:09.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
07:01:09.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:01:09.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_buckets: 2
07:01:09.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) seed: 0
07:01:09.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:01:09.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:01:09.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:09.757083 9587 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table txn_manager_fallback in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
07:01:09.772 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 8df132d56add4b74b452a3573f2f29b2
07:01:09.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.810603 10208 tablet_service.cc:1467] Processing CreateTablet for tablet db85828f290d4e92a96c56d6a71c1de6 (DEFAULT_TABLE table=txn_manager_fallback [id=8df132d56add4b74b452a3573f2f29b2]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
07:01:09.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.810272 10210 tablet_service.cc:1467] Processing CreateTablet for tablet 33dd825246c541e58f1e54d53dce8b17 (DEFAULT_TABLE table=txn_manager_fallback [id=8df132d56add4b74b452a3573f2f29b2]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
07:01:09.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.812434 10208 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet db85828f290d4e92a96c56d6a71c1de6. 1 dirs total, 0 dirs full, 0 dirs failed
07:01:09.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.819540 10210 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 33dd825246c541e58f1e54d53dce8b17. 1 dirs total, 0 dirs full, 0 dirs failed
07:01:09.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.819983 10068 tablet_service.cc:1467] Processing CreateTablet for tablet db85828f290d4e92a96c56d6a71c1de6 (DEFAULT_TABLE table=txn_manager_fallback [id=8df132d56add4b74b452a3573f2f29b2]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
07:01:09.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.821969 10068 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet db85828f290d4e92a96c56d6a71c1de6. 1 dirs total, 0 dirs full, 0 dirs failed
07:01:09.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.818615 10069 tablet_service.cc:1467] Processing CreateTablet for tablet 33dd825246c541e58f1e54d53dce8b17 (DEFAULT_TABLE table=txn_manager_fallback [id=8df132d56add4b74b452a3573f2f29b2]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
07:01:09.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.826522 10069 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 33dd825246c541e58f1e54d53dce8b17. 1 dirs total, 0 dirs full, 0 dirs failed
07:01:09.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.833496 10304 tablet_bootstrap.cc:492] T db85828f290d4e92a96c56d6a71c1de6 P 5289e655d9c647f499e5df5f236ce008: Bootstrap starting.
07:01:09.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.837715 10304 tablet_bootstrap.cc:654] T db85828f290d4e92a96c56d6a71c1de6 P 5289e655d9c647f499e5df5f236ce008: Neither blocks nor log segments found. Creating new log.
07:01:09.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.839286 10304 log.cc:824] T db85828f290d4e92a96c56d6a71c1de6 P 5289e655d9c647f499e5df5f236ce008: Log is configured to *not* fsync() on all Append() calls
07:01:09.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.840030 9929 tablet_service.cc:1467] Processing CreateTablet for tablet db85828f290d4e92a96c56d6a71c1de6 (DEFAULT_TABLE table=txn_manager_fallback [id=8df132d56add4b74b452a3573f2f29b2]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
07:01:09.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.841868 10305 tablet_bootstrap.cc:492] T db85828f290d4e92a96c56d6a71c1de6 P 5d3db908b485446cbcbe6827b7bbbcd7: Bootstrap starting.
07:01:09.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.842068 9929 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet db85828f290d4e92a96c56d6a71c1de6. 1 dirs total, 0 dirs full, 0 dirs failed
07:01:09.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.839995 9930 tablet_service.cc:1467] Processing CreateTablet for tablet 33dd825246c541e58f1e54d53dce8b17 (DEFAULT_TABLE table=txn_manager_fallback [id=8df132d56add4b74b452a3573f2f29b2]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
07:01:09.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.843624 9930 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 33dd825246c541e58f1e54d53dce8b17. 1 dirs total, 0 dirs full, 0 dirs failed
07:01:09.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.846863 10305 tablet_bootstrap.cc:654] T db85828f290d4e92a96c56d6a71c1de6 P 5d3db908b485446cbcbe6827b7bbbcd7: Neither blocks nor log segments found. Creating new log.
07:01:09.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.848404 10305 log.cc:824] T db85828f290d4e92a96c56d6a71c1de6 P 5d3db908b485446cbcbe6827b7bbbcd7: Log is configured to *not* fsync() on all Append() calls
07:01:09.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.848546 10304 tablet_bootstrap.cc:492] T db85828f290d4e92a96c56d6a71c1de6 P 5289e655d9c647f499e5df5f236ce008: No bootstrap required, opened a new log
07:01:09.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.848896 10304 ts_tablet_manager.cc:1367] T db85828f290d4e92a96c56d6a71c1de6 P 5289e655d9c647f499e5df5f236ce008: Time spent bootstrapping tablet: real 0.016s user 0.009s sys 0.004s
07:01:09.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.855110 10305 tablet_bootstrap.cc:492] T db85828f290d4e92a96c56d6a71c1de6 P 5d3db908b485446cbcbe6827b7bbbcd7: No bootstrap required, opened a new log
07:01:09.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.855763 10305 ts_tablet_manager.cc:1367] T db85828f290d4e92a96c56d6a71c1de6 P 5d3db908b485446cbcbe6827b7bbbcd7: Time spent bootstrapping tablet: real 0.014s user 0.009s sys 0.005s
07:01:09.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.858750 10304 raft_consensus.cc:348] T db85828f290d4e92a96c56d6a71c1de6 P 5289e655d9c647f499e5df5f236ce008 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "11ae612cdfd94d638c0c35046ab8118d" member_type: VOTER last_known_addr { host: "127.9.82.65" port: 46241 } } peers { permanent_uuid: "5d3db908b485446cbcbe6827b7bbbcd7" member_type: VOTER last_known_addr { host: "127.9.82.66" port: 37375 } } peers { permanent_uuid: "5289e655d9c647f499e5df5f236ce008" member_type: VOTER last_known_addr { host: "127.9.82.67" port: 36991 } }
07:01:09.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.859546 10304 raft_consensus.cc:374] T db85828f290d4e92a96c56d6a71c1de6 P 5289e655d9c647f499e5df5f236ce008 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:01:09.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.859760 10304 raft_consensus.cc:729] T db85828f290d4e92a96c56d6a71c1de6 P 5289e655d9c647f499e5df5f236ce008 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5289e655d9c647f499e5df5f236ce008, State: Initialized, Role: FOLLOWER
07:01:09.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.860839 10304 consensus_queue.cc:260] T db85828f290d4e92a96c56d6a71c1de6 P 5289e655d9c647f499e5df5f236ce008 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "11ae612cdfd94d638c0c35046ab8118d" member_type: VOTER last_known_addr { host: "127.9.82.65" port: 46241 } } peers { permanent_uuid: "5d3db908b485446cbcbe6827b7bbbcd7" member_type: VOTER last_known_addr { host: "127.9.82.66" port: 37375 } } peers { permanent_uuid: "5289e655d9c647f499e5df5f236ce008" member_type: VOTER last_known_addr { host: "127.9.82.67" port: 36991 } }
07:01:09.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.861387 10308 tablet_bootstrap.cc:492] T db85828f290d4e92a96c56d6a71c1de6 P 11ae612cdfd94d638c0c35046ab8118d: Bootstrap starting.
07:01:09.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.866019 10308 tablet_bootstrap.cc:654] T db85828f290d4e92a96c56d6a71c1de6 P 11ae612cdfd94d638c0c35046ab8118d: Neither blocks nor log segments found. Creating new log.
07:01:09.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.866353 10305 raft_consensus.cc:348] T db85828f290d4e92a96c56d6a71c1de6 P 5d3db908b485446cbcbe6827b7bbbcd7 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "11ae612cdfd94d638c0c35046ab8118d" member_type: VOTER last_known_addr { host: "127.9.82.65" port: 46241 } } peers { permanent_uuid: "5d3db908b485446cbcbe6827b7bbbcd7" member_type: VOTER last_known_addr { host: "127.9.82.66" port: 37375 } } peers { permanent_uuid: "5289e655d9c647f499e5df5f236ce008" member_type: VOTER last_known_addr { host: "127.9.82.67" port: 36991 } }
07:01:09.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.867468 10305 raft_consensus.cc:374] T db85828f290d4e92a96c56d6a71c1de6 P 5d3db908b485446cbcbe6827b7bbbcd7 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:01:09.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.867733 10305 raft_consensus.cc:729] T db85828f290d4e92a96c56d6a71c1de6 P 5d3db908b485446cbcbe6827b7bbbcd7 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5d3db908b485446cbcbe6827b7bbbcd7, State: Initialized, Role: FOLLOWER
07:01:09.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.869976 10304 ts_tablet_manager.cc:1398] T db85828f290d4e92a96c56d6a71c1de6 P 5289e655d9c647f499e5df5f236ce008: Time spent starting tablet: real 0.021s user 0.011s sys 0.008s
07:01:09.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.870517 10279 heartbeater.cc:503] Master 127.9.82.126:43671 was elected leader, sending a full tablet report...
07:01:09.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.870612 10308 log.cc:824] T db85828f290d4e92a96c56d6a71c1de6 P 11ae612cdfd94d638c0c35046ab8118d: Log is configured to *not* fsync() on all Append() calls
07:01:09.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.875087 10305 consensus_queue.cc:260] T db85828f290d4e92a96c56d6a71c1de6 P 5d3db908b485446cbcbe6827b7bbbcd7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "11ae612cdfd94d638c0c35046ab8118d" member_type: VOTER last_known_addr { host: "127.9.82.65" port: 46241 } } peers { permanent_uuid: "5d3db908b485446cbcbe6827b7bbbcd7" member_type: VOTER last_known_addr { host: "127.9.82.66" port: 37375 } } peers { permanent_uuid: "5289e655d9c647f499e5df5f236ce008" member_type: VOTER last_known_addr { host: "127.9.82.67" port: 36991 } }
07:01:09.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.881596 10304 tablet_bootstrap.cc:492] T 33dd825246c541e58f1e54d53dce8b17 P 5289e655d9c647f499e5df5f236ce008: Bootstrap starting.
07:01:09.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.884549 10308 tablet_bootstrap.cc:492] T db85828f290d4e92a96c56d6a71c1de6 P 11ae612cdfd94d638c0c35046ab8118d: No bootstrap required, opened a new log
07:01:09.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.885159 10308 ts_tablet_manager.cc:1367] T db85828f290d4e92a96c56d6a71c1de6 P 11ae612cdfd94d638c0c35046ab8118d: Time spent bootstrapping tablet: real 0.024s user 0.013s sys 0.007s
07:01:09.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.887758 10304 tablet_bootstrap.cc:654] T 33dd825246c541e58f1e54d53dce8b17 P 5289e655d9c647f499e5df5f236ce008: Neither blocks nor log segments found. Creating new log.
07:01:09.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.891633 10305 ts_tablet_manager.cc:1398] T db85828f290d4e92a96c56d6a71c1de6 P 5d3db908b485446cbcbe6827b7bbbcd7: Time spent starting tablet: real 0.036s user 0.019s sys 0.009s
07:01:09.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.892202 10138 heartbeater.cc:503] Master 127.9.82.126:43671 was elected leader, sending a full tablet report...
07:01:09.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.893875 10305 tablet_bootstrap.cc:492] T 33dd825246c541e58f1e54d53dce8b17 P 5d3db908b485446cbcbe6827b7bbbcd7: Bootstrap starting.
07:01:09.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.895895 10305 tablet_bootstrap.cc:654] T 33dd825246c541e58f1e54d53dce8b17 P 5d3db908b485446cbcbe6827b7bbbcd7: Neither blocks nor log segments found. Creating new log.
07:01:09.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.896696 10308 raft_consensus.cc:348] T db85828f290d4e92a96c56d6a71c1de6 P 11ae612cdfd94d638c0c35046ab8118d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "11ae612cdfd94d638c0c35046ab8118d" member_type: VOTER last_known_addr { host: "127.9.82.65" port: 46241 } } peers { permanent_uuid: "5d3db908b485446cbcbe6827b7bbbcd7" member_type: VOTER last_known_addr { host: "127.9.82.66" port: 37375 } } peers { permanent_uuid: "5289e655d9c647f499e5df5f236ce008" member_type: VOTER last_known_addr { host: "127.9.82.67" port: 36991 } }
07:01:09.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.897300 10308 raft_consensus.cc:374] T db85828f290d4e92a96c56d6a71c1de6 P 11ae612cdfd94d638c0c35046ab8118d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:01:09.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.897516 10308 raft_consensus.cc:729] T db85828f290d4e92a96c56d6a71c1de6 P 11ae612cdfd94d638c0c35046ab8118d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 11ae612cdfd94d638c0c35046ab8118d, State: Initialized, Role: FOLLOWER
07:01:09.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.898591 10308 consensus_queue.cc:260] T db85828f290d4e92a96c56d6a71c1de6 P 11ae612cdfd94d638c0c35046ab8118d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "11ae612cdfd94d638c0c35046ab8118d" member_type: VOTER last_known_addr { host: "127.9.82.65" port: 46241 } } peers { permanent_uuid: "5d3db908b485446cbcbe6827b7bbbcd7" member_type: VOTER last_known_addr { host: "127.9.82.66" port: 37375 } } peers { permanent_uuid: "5289e655d9c647f499e5df5f236ce008" member_type: VOTER last_known_addr { host: "127.9.82.67" port: 36991 } }
07:01:09.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.903235 10304 tablet_bootstrap.cc:492] T 33dd825246c541e58f1e54d53dce8b17 P 5289e655d9c647f499e5df5f236ce008: No bootstrap required, opened a new log
07:01:09.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.903491 10304 ts_tablet_manager.cc:1367] T 33dd825246c541e58f1e54d53dce8b17 P 5289e655d9c647f499e5df5f236ce008: Time spent bootstrapping tablet: real 0.022s user 0.008s sys 0.000s
07:01:09.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.903913 10308 ts_tablet_manager.cc:1398] T db85828f290d4e92a96c56d6a71c1de6 P 11ae612cdfd94d638c0c35046ab8118d: Time spent starting tablet: real 0.019s user 0.015s sys 0.000s
07:01:09.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.904958 10308 tablet_bootstrap.cc:492] T 33dd825246c541e58f1e54d53dce8b17 P 11ae612cdfd94d638c0c35046ab8118d: Bootstrap starting.
07:01:09.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.905885 10305 tablet_bootstrap.cc:492] T 33dd825246c541e58f1e54d53dce8b17 P 5d3db908b485446cbcbe6827b7bbbcd7: No bootstrap required, opened a new log
07:01:09.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.906229 10305 ts_tablet_manager.cc:1367] T 33dd825246c541e58f1e54d53dce8b17 P 5d3db908b485446cbcbe6827b7bbbcd7: Time spent bootstrapping tablet: real 0.012s user 0.005s sys 0.004s
07:01:09.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.907155 10305 raft_consensus.cc:348] T 33dd825246c541e58f1e54d53dce8b17 P 5d3db908b485446cbcbe6827b7bbbcd7 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5289e655d9c647f499e5df5f236ce008" member_type: VOTER last_known_addr { host: "127.9.82.67" port: 36991 } } peers { permanent_uuid: "5d3db908b485446cbcbe6827b7bbbcd7" member_type: VOTER last_known_addr { host: "127.9.82.66" port: 37375 } } peers { permanent_uuid: "11ae612cdfd94d638c0c35046ab8118d" member_type: VOTER last_known_addr { host: "127.9.82.65" port: 46241 } }
07:01:09.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.907501 10305 raft_consensus.cc:374] T 33dd825246c541e58f1e54d53dce8b17 P 5d3db908b485446cbcbe6827b7bbbcd7 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:01:09.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.907681 10305 raft_consensus.cc:729] T 33dd825246c541e58f1e54d53dce8b17 P 5d3db908b485446cbcbe6827b7bbbcd7 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5d3db908b485446cbcbe6827b7bbbcd7, State: Initialized, Role: FOLLOWER
07:01:09.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.908047 10305 consensus_queue.cc:260] T 33dd825246c541e58f1e54d53dce8b17 P 5d3db908b485446cbcbe6827b7bbbcd7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5289e655d9c647f499e5df5f236ce008" member_type: VOTER last_known_addr { host: "127.9.82.67" port: 36991 } } peers { permanent_uuid: "5d3db908b485446cbcbe6827b7bbbcd7" member_type: VOTER last_known_addr { host: "127.9.82.66" port: 37375 } } peers { permanent_uuid: "11ae612cdfd94d638c0c35046ab8118d" member_type: VOTER last_known_addr { host: "127.9.82.65" port: 46241 } }
07:01:09.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.908494 10308 tablet_bootstrap.cc:654] T 33dd825246c541e58f1e54d53dce8b17 P 11ae612cdfd94d638c0c35046ab8118d: Neither blocks nor log segments found. Creating new log.
07:01:09.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.908659 10305 ts_tablet_manager.cc:1398] T 33dd825246c541e58f1e54d53dce8b17 P 5d3db908b485446cbcbe6827b7bbbcd7: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
07:01:09.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.904393 10304 raft_consensus.cc:348] T 33dd825246c541e58f1e54d53dce8b17 P 5289e655d9c647f499e5df5f236ce008 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5289e655d9c647f499e5df5f236ce008" member_type: VOTER last_known_addr { host: "127.9.82.67" port: 36991 } } peers { permanent_uuid: "5d3db908b485446cbcbe6827b7bbbcd7" member_type: VOTER last_known_addr { host: "127.9.82.66" port: 37375 } } peers { permanent_uuid: "11ae612cdfd94d638c0c35046ab8118d" member_type: VOTER last_known_addr { host: "127.9.82.65" port: 46241 } }
07:01:09.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.910952 10304 raft_consensus.cc:374] T 33dd825246c541e58f1e54d53dce8b17 P 5289e655d9c647f499e5df5f236ce008 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:01:09.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.911162 10304 raft_consensus.cc:729] T 33dd825246c541e58f1e54d53dce8b17 P 5289e655d9c647f499e5df5f236ce008 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5289e655d9c647f499e5df5f236ce008, State: Initialized, Role: FOLLOWER
07:01:09.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.911608 10304 consensus_queue.cc:260] T 33dd825246c541e58f1e54d53dce8b17 P 5289e655d9c647f499e5df5f236ce008 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5289e655d9c647f499e5df5f236ce008" member_type: VOTER last_known_addr { host: "127.9.82.67" port: 36991 } } peers { permanent_uuid: "5d3db908b485446cbcbe6827b7bbbcd7" member_type: VOTER last_known_addr { host: "127.9.82.66" port: 37375 } } peers { permanent_uuid: "11ae612cdfd94d638c0c35046ab8118d" member_type: VOTER last_known_addr { host: "127.9.82.65" port: 46241 } }
07:01:09.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.913326 10304 ts_tablet_manager.cc:1398] T 33dd825246c541e58f1e54d53dce8b17 P 5289e655d9c647f499e5df5f236ce008: Time spent starting tablet: real 0.010s user 0.002s sys 0.000s
07:01:09.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.917304 10308 tablet_bootstrap.cc:492] T 33dd825246c541e58f1e54d53dce8b17 P 11ae612cdfd94d638c0c35046ab8118d: No bootstrap required, opened a new log
07:01:09.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.917452 10308 ts_tablet_manager.cc:1367] T 33dd825246c541e58f1e54d53dce8b17 P 11ae612cdfd94d638c0c35046ab8118d: Time spent bootstrapping tablet: real 0.013s user 0.004s sys 0.004s
07:01:09.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.918081 10308 raft_consensus.cc:348] T 33dd825246c541e58f1e54d53dce8b17 P 11ae612cdfd94d638c0c35046ab8118d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5289e655d9c647f499e5df5f236ce008" member_type: VOTER last_known_addr { host: "127.9.82.67" port: 36991 } } peers { permanent_uuid: "5d3db908b485446cbcbe6827b7bbbcd7" member_type: VOTER last_known_addr { host: "127.9.82.66" port: 37375 } } peers { permanent_uuid: "11ae612cdfd94d638c0c35046ab8118d" member_type: VOTER last_known_addr { host: "127.9.82.65" port: 46241 } }
07:01:09.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.918244 10308 raft_consensus.cc:374] T 33dd825246c541e58f1e54d53dce8b17 P 11ae612cdfd94d638c0c35046ab8118d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:01:09.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.918299 10308 raft_consensus.cc:729] T 33dd825246c541e58f1e54d53dce8b17 P 11ae612cdfd94d638c0c35046ab8118d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 11ae612cdfd94d638c0c35046ab8118d, State: Initialized, Role: FOLLOWER
07:01:09.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.918484 10308 consensus_queue.cc:260] T 33dd825246c541e58f1e54d53dce8b17 P 11ae612cdfd94d638c0c35046ab8118d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5289e655d9c647f499e5df5f236ce008" member_type: VOTER last_known_addr { host: "127.9.82.67" port: 36991 } } peers { permanent_uuid: "5d3db908b485446cbcbe6827b7bbbcd7" member_type: VOTER last_known_addr { host: "127.9.82.66" port: 37375 } } peers { permanent_uuid: "11ae612cdfd94d638c0c35046ab8118d" member_type: VOTER last_known_addr { host: "127.9.82.65" port: 46241 } }
07:01:09.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:09.918946 10308 ts_tablet_manager.cc:1398] T 33dd825246c541e58f1e54d53dce8b17 P 11ae612cdfd94d638c0c35046ab8118d: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
07:01:09.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:09.947574 10140 tablet.cc:2369] T 33dd825246c541e58f1e54d53dce8b17 P 5d3db908b485446cbcbe6827b7bbbcd7: Can't schedule compaction. Clean time has not been advanced past its initial value.
07:01:09.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:09.956687 10281 tablet.cc:2369] T 33dd825246c541e58f1e54d53dce8b17 P 5289e655d9c647f499e5df5f236ce008: Can't schedule compaction. Clean time has not been advanced past its initial value.
07:01:10.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.010835 10311 raft_consensus.cc:482] T db85828f290d4e92a96c56d6a71c1de6 P 5d3db908b485446cbcbe6827b7bbbcd7 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:01:10.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.011163 10311 raft_consensus.cc:504] T db85828f290d4e92a96c56d6a71c1de6 P 5d3db908b485446cbcbe6827b7bbbcd7 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "11ae612cdfd94d638c0c35046ab8118d" member_type: VOTER last_known_addr { host: "127.9.82.65" port: 46241 } } peers { permanent_uuid: "5d3db908b485446cbcbe6827b7bbbcd7" member_type: VOTER last_known_addr { host: "127.9.82.66" port: 37375 } } peers { permanent_uuid: "5289e655d9c647f499e5df5f236ce008" member_type: VOTER last_known_addr { host: "127.9.82.67" port: 36991 } }
07:01:10.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.013434 10311 leader_election.cc:288] T db85828f290d4e92a96c56d6a71c1de6 P 5d3db908b485446cbcbe6827b7bbbcd7 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 11ae612cdfd94d638c0c35046ab8118d (127.9.82.65:46241), 5289e655d9c647f499e5df5f236ce008 (127.9.82.67:36991)
07:01:10.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.020546 10233 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "db85828f290d4e92a96c56d6a71c1de6" candidate_uuid: "5d3db908b485446cbcbe6827b7bbbcd7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5289e655d9c647f499e5df5f236ce008" is_pre_election: true
07:01:10.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.020547 9950 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "db85828f290d4e92a96c56d6a71c1de6" candidate_uuid: "5d3db908b485446cbcbe6827b7bbbcd7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "11ae612cdfd94d638c0c35046ab8118d" is_pre_election: true
07:01:10.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.021097 10233 raft_consensus.cc:2454] T db85828f290d4e92a96c56d6a71c1de6 P 5289e655d9c647f499e5df5f236ce008 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5d3db908b485446cbcbe6827b7bbbcd7 in term 0.
07:01:10.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.021097 9950 raft_consensus.cc:2454] T db85828f290d4e92a96c56d6a71c1de6 P 11ae612cdfd94d638c0c35046ab8118d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5d3db908b485446cbcbe6827b7bbbcd7 in term 0.
07:01:10.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.021791 10017 leader_election.cc:302] T db85828f290d4e92a96c56d6a71c1de6 P 5d3db908b485446cbcbe6827b7bbbcd7 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 11ae612cdfd94d638c0c35046ab8118d, 5d3db908b485446cbcbe6827b7bbbcd7; no voters:
07:01:10.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.022202 10311 raft_consensus.cc:2781] T db85828f290d4e92a96c56d6a71c1de6 P 5d3db908b485446cbcbe6827b7bbbcd7 [term 0 FOLLOWER]: Leader pre-election won for term 1
07:01:10.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.022339 10311 raft_consensus.cc:482] T db85828f290d4e92a96c56d6a71c1de6 P 5d3db908b485446cbcbe6827b7bbbcd7 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
07:01:10.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.022415 10311 raft_consensus.cc:3037] T db85828f290d4e92a96c56d6a71c1de6 P 5d3db908b485446cbcbe6827b7bbbcd7 [term 0 FOLLOWER]: Advancing to term 1
07:01:10.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.024582 10311 raft_consensus.cc:504] T db85828f290d4e92a96c56d6a71c1de6 P 5d3db908b485446cbcbe6827b7bbbcd7 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "11ae612cdfd94d638c0c35046ab8118d" member_type: VOTER last_known_addr { host: "127.9.82.65" port: 46241 } } peers { permanent_uuid: "5d3db908b485446cbcbe6827b7bbbcd7" member_type: VOTER last_known_addr { host: "127.9.82.66" port: 37375 } } peers { permanent_uuid: "5289e655d9c647f499e5df5f236ce008" member_type: VOTER last_known_addr { host: "127.9.82.67" port: 36991 } }
07:01:10.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.025202 10311 leader_election.cc:288] T db85828f290d4e92a96c56d6a71c1de6 P 5d3db908b485446cbcbe6827b7bbbcd7 [CANDIDATE]: Term 1 election: Requested vote from peers 11ae612cdfd94d638c0c35046ab8118d (127.9.82.65:46241), 5289e655d9c647f499e5df5f236ce008 (127.9.82.67:36991)
07:01:10.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.025853 10233 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "db85828f290d4e92a96c56d6a71c1de6" candidate_uuid: "5d3db908b485446cbcbe6827b7bbbcd7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5289e655d9c647f499e5df5f236ce008"
07:01:10.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.025853 9950 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "db85828f290d4e92a96c56d6a71c1de6" candidate_uuid: "5d3db908b485446cbcbe6827b7bbbcd7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "11ae612cdfd94d638c0c35046ab8118d"
07:01:10.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.026134 9950 raft_consensus.cc:3037] T db85828f290d4e92a96c56d6a71c1de6 P 11ae612cdfd94d638c0c35046ab8118d [term 0 FOLLOWER]: Advancing to term 1
07:01:10.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.026135 10233 raft_consensus.cc:3037] T db85828f290d4e92a96c56d6a71c1de6 P 5289e655d9c647f499e5df5f236ce008 [term 0 FOLLOWER]: Advancing to term 1
07:01:10.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.029433 9950 raft_consensus.cc:2454] T db85828f290d4e92a96c56d6a71c1de6 P 11ae612cdfd94d638c0c35046ab8118d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5d3db908b485446cbcbe6827b7bbbcd7 in term 1.
07:01:10.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.029433 10233 raft_consensus.cc:2454] T db85828f290d4e92a96c56d6a71c1de6 P 5289e655d9c647f499e5df5f236ce008 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5d3db908b485446cbcbe6827b7bbbcd7 in term 1.
07:01:10.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.030071 10016 leader_election.cc:302] T db85828f290d4e92a96c56d6a71c1de6 P 5d3db908b485446cbcbe6827b7bbbcd7 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 5289e655d9c647f499e5df5f236ce008, 5d3db908b485446cbcbe6827b7bbbcd7; no voters:
07:01:10.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.030447 10311 raft_consensus.cc:2781] T db85828f290d4e92a96c56d6a71c1de6 P 5d3db908b485446cbcbe6827b7bbbcd7 [term 1 FOLLOWER]: Leader election won for term 1
07:01:10.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.030944 10311 raft_consensus.cc:686] T db85828f290d4e92a96c56d6a71c1de6 P 5d3db908b485446cbcbe6827b7bbbcd7 [term 1 LEADER]: Becoming Leader. State: Replica: 5d3db908b485446cbcbe6827b7bbbcd7, State: Running, Role: LEADER
07:01:10.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.031755 10311 consensus_queue.cc:237] T db85828f290d4e92a96c56d6a71c1de6 P 5d3db908b485446cbcbe6827b7bbbcd7 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "11ae612cdfd94d638c0c35046ab8118d" member_type: VOTER last_known_addr { host: "127.9.82.65" port: 46241 } } peers { permanent_uuid: "5d3db908b485446cbcbe6827b7bbbcd7" member_type: VOTER last_known_addr { host: "127.9.82.66" port: 37375 } } peers { permanent_uuid: "5289e655d9c647f499e5df5f236ce008" member_type: VOTER last_known_addr { host: "127.9.82.67" port: 36991 } }
07:01:10.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.038431 9586 catalog_manager.cc:5484] T db85828f290d4e92a96c56d6a71c1de6 P 5d3db908b485446cbcbe6827b7bbbcd7 reported cstate change: term changed from 0 to 1, leader changed from <none> to 5d3db908b485446cbcbe6827b7bbbcd7 (127.9.82.66). New cstate: current_term: 1 leader_uuid: "5d3db908b485446cbcbe6827b7bbbcd7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "11ae612cdfd94d638c0c35046ab8118d" member_type: VOTER last_known_addr { host: "127.9.82.65" port: 46241 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5d3db908b485446cbcbe6827b7bbbcd7" member_type: VOTER last_known_addr { host: "127.9.82.66" port: 37375 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5289e655d9c647f499e5df5f236ce008" member_type: VOTER last_known_addr { host: "127.9.82.67" port: 36991 } health_report { overall_health: UNKNOWN } } }
07:01:10.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.044404 10309 raft_consensus.cc:482] T 33dd825246c541e58f1e54d53dce8b17 P 5289e655d9c647f499e5df5f236ce008 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:01:10.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.044646 10309 raft_consensus.cc:504] T 33dd825246c541e58f1e54d53dce8b17 P 5289e655d9c647f499e5df5f236ce008 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5289e655d9c647f499e5df5f236ce008" member_type: VOTER last_known_addr { host: "127.9.82.67" port: 36991 } } peers { permanent_uuid: "5d3db908b485446cbcbe6827b7bbbcd7" member_type: VOTER last_known_addr { host: "127.9.82.66" port: 37375 } } peers { permanent_uuid: "11ae612cdfd94d638c0c35046ab8118d" member_type: VOTER last_known_addr { host: "127.9.82.65" port: 46241 } }
07:01:10.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.047395 10309 leader_election.cc:288] T 33dd825246c541e58f1e54d53dce8b17 P 5289e655d9c647f499e5df5f236ce008 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5d3db908b485446cbcbe6827b7bbbcd7 (127.9.82.66:37375), 11ae612cdfd94d638c0c35046ab8118d (127.9.82.65:46241)
07:01:10.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.053673 9950 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "33dd825246c541e58f1e54d53dce8b17" candidate_uuid: "5289e655d9c647f499e5df5f236ce008" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "11ae612cdfd94d638c0c35046ab8118d" is_pre_election: true
07:01:10.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.053937 9950 raft_consensus.cc:2454] T 33dd825246c541e58f1e54d53dce8b17 P 11ae612cdfd94d638c0c35046ab8118d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5289e655d9c647f499e5df5f236ce008 in term 0.
07:01:10.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.053952 10092 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "33dd825246c541e58f1e54d53dce8b17" candidate_uuid: "5289e655d9c647f499e5df5f236ce008" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5d3db908b485446cbcbe6827b7bbbcd7" is_pre_election: true
07:01:10.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.054526 10092 raft_consensus.cc:2454] T 33dd825246c541e58f1e54d53dce8b17 P 5d3db908b485446cbcbe6827b7bbbcd7 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5289e655d9c647f499e5df5f236ce008 in term 0.
07:01:10.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.054507 10158 leader_election.cc:302] T 33dd825246c541e58f1e54d53dce8b17 P 5289e655d9c647f499e5df5f236ce008 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 11ae612cdfd94d638c0c35046ab8118d, 5289e655d9c647f499e5df5f236ce008; no voters:
07:01:10.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.055130 10309 raft_consensus.cc:2781] T 33dd825246c541e58f1e54d53dce8b17 P 5289e655d9c647f499e5df5f236ce008 [term 0 FOLLOWER]: Leader pre-election won for term 1
07:01:10.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.055296 10309 raft_consensus.cc:482] T 33dd825246c541e58f1e54d53dce8b17 P 5289e655d9c647f499e5df5f236ce008 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
07:01:10.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.055397 10309 raft_consensus.cc:3037] T 33dd825246c541e58f1e54d53dce8b17 P 5289e655d9c647f499e5df5f236ce008 [term 0 FOLLOWER]: Advancing to term 1
07:01:10.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.057456 10309 raft_consensus.cc:504] T 33dd825246c541e58f1e54d53dce8b17 P 5289e655d9c647f499e5df5f236ce008 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5289e655d9c647f499e5df5f236ce008" member_type: VOTER last_known_addr { host: "127.9.82.67" port: 36991 } } peers { permanent_uuid: "5d3db908b485446cbcbe6827b7bbbcd7" member_type: VOTER last_known_addr { host: "127.9.82.66" port: 37375 } } peers { permanent_uuid: "11ae612cdfd94d638c0c35046ab8118d" member_type: VOTER last_known_addr { host: "127.9.82.65" port: 46241 } }
07:01:10.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.058347 10309 leader_election.cc:288] T 33dd825246c541e58f1e54d53dce8b17 P 5289e655d9c647f499e5df5f236ce008 [CANDIDATE]: Term 1 election: Requested vote from peers 5d3db908b485446cbcbe6827b7bbbcd7 (127.9.82.66:37375), 11ae612cdfd94d638c0c35046ab8118d (127.9.82.65:46241)
07:01:10.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.058861 10092 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "33dd825246c541e58f1e54d53dce8b17" candidate_uuid: "5289e655d9c647f499e5df5f236ce008" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5d3db908b485446cbcbe6827b7bbbcd7"
07:01:10.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.058838 9950 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "33dd825246c541e58f1e54d53dce8b17" candidate_uuid: "5289e655d9c647f499e5df5f236ce008" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "11ae612cdfd94d638c0c35046ab8118d"
07:01:10.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.059085 10092 raft_consensus.cc:3037] T 33dd825246c541e58f1e54d53dce8b17 P 5d3db908b485446cbcbe6827b7bbbcd7 [term 0 FOLLOWER]: Advancing to term 1
07:01:10.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.059104 9950 raft_consensus.cc:3037] T 33dd825246c541e58f1e54d53dce8b17 P 11ae612cdfd94d638c0c35046ab8118d [term 0 FOLLOWER]: Advancing to term 1
07:01:10.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.062286 9950 raft_consensus.cc:2454] T 33dd825246c541e58f1e54d53dce8b17 P 11ae612cdfd94d638c0c35046ab8118d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5289e655d9c647f499e5df5f236ce008 in term 1.
07:01:10.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.062312 10092 raft_consensus.cc:2454] T 33dd825246c541e58f1e54d53dce8b17 P 5d3db908b485446cbcbe6827b7bbbcd7 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5289e655d9c647f499e5df5f236ce008 in term 1.
07:01:10.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.062922 10157 leader_election.cc:302] T 33dd825246c541e58f1e54d53dce8b17 P 5289e655d9c647f499e5df5f236ce008 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 5289e655d9c647f499e5df5f236ce008, 5d3db908b485446cbcbe6827b7bbbcd7; no voters:
07:01:10.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.063463 10309 raft_consensus.cc:2781] T 33dd825246c541e58f1e54d53dce8b17 P 5289e655d9c647f499e5df5f236ce008 [term 1 FOLLOWER]: Leader election won for term 1
07:01:10.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.063906 10309 raft_consensus.cc:686] T 33dd825246c541e58f1e54d53dce8b17 P 5289e655d9c647f499e5df5f236ce008 [term 1 LEADER]: Becoming Leader. State: Replica: 5289e655d9c647f499e5df5f236ce008, State: Running, Role: LEADER
07:01:10.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.064528 10309 consensus_queue.cc:237] T 33dd825246c541e58f1e54d53dce8b17 P 5289e655d9c647f499e5df5f236ce008 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5289e655d9c647f499e5df5f236ce008" member_type: VOTER last_known_addr { host: "127.9.82.67" port: 36991 } } peers { permanent_uuid: "5d3db908b485446cbcbe6827b7bbbcd7" member_type: VOTER last_known_addr { host: "127.9.82.66" port: 37375 } } peers { permanent_uuid: "11ae612cdfd94d638c0c35046ab8118d" member_type: VOTER last_known_addr { host: "127.9.82.65" port: 46241 } }
07:01:10.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.071265 9586 catalog_manager.cc:5484] T 33dd825246c541e58f1e54d53dce8b17 P 5289e655d9c647f499e5df5f236ce008 reported cstate change: term changed from 0 to 1, leader changed from <none> to 5289e655d9c647f499e5df5f236ce008 (127.9.82.67). New cstate: current_term: 1 leader_uuid: "5289e655d9c647f499e5df5f236ce008" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5289e655d9c647f499e5df5f236ce008" member_type: VOTER last_known_addr { host: "127.9.82.67" port: 36991 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5d3db908b485446cbcbe6827b7bbbcd7" member_type: VOTER last_known_addr { host: "127.9.82.66" port: 37375 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "11ae612cdfd94d638c0c35046ab8118d" member_type: VOTER last_known_addr { host: "127.9.82.65" port: 46241 } health_report { overall_health: UNKNOWN } } }
07:01:10.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.128507 9586 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:35422:
07:01:10.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
07:01:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
07:01:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:01:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
07:01:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
07:01:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
07:01:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:01:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
07:01:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
07:01:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:01:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:01:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:01:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:01:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
07:01:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
07:01:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
07:01:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:01:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
07:01:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
07:01:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:01:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:01:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:01:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:01:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
07:01:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
07:01:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
07:01:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:01:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
07:01:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
07:01:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:01:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:01:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:01:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:01:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
07:01:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
07:01:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:01:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:01:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
07:01:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
07:01:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:01:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:01:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:01:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:01:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
07:01:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
07:01:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
07:01:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
07:01:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:01:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
07:01:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
07:01:10.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:01:10.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
07:01:10.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:01:10.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:01:10.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:01:10.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
07:01:10.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:10.129712 9586 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
07:01:10.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.141153 10208 tablet_service.cc:1467] Processing CreateTablet for tablet fef62f88ed7a42d9b3f16fcaa8ac2250 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=c7e9c9d4914946a2b4e4b9116312e2cb]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
07:01:10.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.141081 10069 tablet_service.cc:1467] Processing CreateTablet for tablet fef62f88ed7a42d9b3f16fcaa8ac2250 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=c7e9c9d4914946a2b4e4b9116312e2cb]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
07:01:10.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.141175 9929 tablet_service.cc:1467] Processing CreateTablet for tablet fef62f88ed7a42d9b3f16fcaa8ac2250 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=c7e9c9d4914946a2b4e4b9116312e2cb]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
07:01:10.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.142123 9929 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fef62f88ed7a42d9b3f16fcaa8ac2250. 1 dirs total, 0 dirs full, 0 dirs failed
07:01:10.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.142123 10069 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fef62f88ed7a42d9b3f16fcaa8ac2250. 1 dirs total, 0 dirs full, 0 dirs failed
07:01:10.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.142141 10208 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fef62f88ed7a42d9b3f16fcaa8ac2250. 1 dirs total, 0 dirs full, 0 dirs failed
07:01:10.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.147476 10304 tablet_bootstrap.cc:492] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 5289e655d9c647f499e5df5f236ce008: Bootstrap starting.
07:01:10.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.149046 10308 tablet_bootstrap.cc:492] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 11ae612cdfd94d638c0c35046ab8118d: Bootstrap starting.
07:01:10.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.149201 10304 tablet_bootstrap.cc:654] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 5289e655d9c647f499e5df5f236ce008: Neither blocks nor log segments found. Creating new log.
07:01:10.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.150532 10308 tablet_bootstrap.cc:654] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 11ae612cdfd94d638c0c35046ab8118d: Neither blocks nor log segments found. Creating new log.
07:01:10.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.150575 10305 tablet_bootstrap.cc:492] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 5d3db908b485446cbcbe6827b7bbbcd7: Bootstrap starting.
07:01:10.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:10.151602 9999 tablet.cc:2369] T 33dd825246c541e58f1e54d53dce8b17 P 11ae612cdfd94d638c0c35046ab8118d: Can't schedule compaction. Clean time has not been advanced past its initial value.
07:01:10.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.152510 10305 tablet_bootstrap.cc:654] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 5d3db908b485446cbcbe6827b7bbbcd7: Neither blocks nor log segments found. Creating new log.
07:01:10.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.152664 10304 tablet_bootstrap.cc:492] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 5289e655d9c647f499e5df5f236ce008: No bootstrap required, opened a new log
07:01:10.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.152953 10304 ts_tablet_manager.cc:1367] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 5289e655d9c647f499e5df5f236ce008: Time spent bootstrapping tablet: real 0.005s user 0.003s sys 0.001s
07:01:10.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.153760 10304 raft_consensus.cc:348] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 5289e655d9c647f499e5df5f236ce008 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "11ae612cdfd94d638c0c35046ab8118d" member_type: VOTER last_known_addr { host: "127.9.82.65" port: 46241 } } peers { permanent_uuid: "5d3db908b485446cbcbe6827b7bbbcd7" member_type: VOTER last_known_addr { host: "127.9.82.66" port: 37375 } } peers { permanent_uuid: "5289e655d9c647f499e5df5f236ce008" member_type: VOTER last_known_addr { host: "127.9.82.67" port: 36991 } }
07:01:10.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.154011 10304 raft_consensus.cc:374] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 5289e655d9c647f499e5df5f236ce008 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:01:10.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.154083 10304 raft_consensus.cc:729] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 5289e655d9c647f499e5df5f236ce008 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5289e655d9c647f499e5df5f236ce008, State: Initialized, Role: FOLLOWER
07:01:10.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.154367 10304 consensus_queue.cc:260] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 5289e655d9c647f499e5df5f236ce008 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "11ae612cdfd94d638c0c35046ab8118d" member_type: VOTER last_known_addr { host: "127.9.82.65" port: 46241 } } peers { permanent_uuid: "5d3db908b485446cbcbe6827b7bbbcd7" member_type: VOTER last_known_addr { host: "127.9.82.66" port: 37375 } } peers { permanent_uuid: "5289e655d9c647f499e5df5f236ce008" member_type: VOTER last_known_addr { host: "127.9.82.67" port: 36991 } }
07:01:10.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.154908 10309 tablet_replica.cc:439] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 5289e655d9c647f499e5df5f236ce008: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "11ae612cdfd94d638c0c35046ab8118d" member_type: VOTER last_known_addr { host: "127.9.82.65" port: 46241 } } peers { permanent_uuid: "5d3db908b485446cbcbe6827b7bbbcd7" member_type: VOTER last_known_addr { host: "127.9.82.66" port: 37375 } } peers { permanent_uuid: "5289e655d9c647f499e5df5f236ce008" member_type: VOTER last_known_addr { host: "127.9.82.67" port: 36991 } } }
07:01:10.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.155177 10309 tablet_replica.cc:442] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 5289e655d9c647f499e5df5f236ce008: This TxnStatusTablet replica's current role is: FOLLOWER
07:01:10.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.155656 10304 ts_tablet_manager.cc:1398] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 5289e655d9c647f499e5df5f236ce008: Time spent starting tablet: real 0.003s user 0.002s sys 0.001s
07:01:10.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.160233 10308 tablet_bootstrap.cc:492] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 11ae612cdfd94d638c0c35046ab8118d: No bootstrap required, opened a new log
07:01:10.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.160491 10308 ts_tablet_manager.cc:1367] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 11ae612cdfd94d638c0c35046ab8118d: Time spent bootstrapping tablet: real 0.011s user 0.009s sys 0.000s
07:01:10.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.161465 10308 raft_consensus.cc:348] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 11ae612cdfd94d638c0c35046ab8118d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "11ae612cdfd94d638c0c35046ab8118d" member_type: VOTER last_known_addr { host: "127.9.82.65" port: 46241 } } peers { permanent_uuid: "5d3db908b485446cbcbe6827b7bbbcd7" member_type: VOTER last_known_addr { host: "127.9.82.66" port: 37375 } } peers { permanent_uuid: "5289e655d9c647f499e5df5f236ce008" member_type: VOTER last_known_addr { host: "127.9.82.67" port: 36991 } }
07:01:10.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.161768 10308 raft_consensus.cc:374] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 11ae612cdfd94d638c0c35046ab8118d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:01:10.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.161854 10308 raft_consensus.cc:729] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 11ae612cdfd94d638c0c35046ab8118d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 11ae612cdfd94d638c0c35046ab8118d, State: Initialized, Role: FOLLOWER
07:01:10.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.162267 10308 consensus_queue.cc:260] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 11ae612cdfd94d638c0c35046ab8118d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "11ae612cdfd94d638c0c35046ab8118d" member_type: VOTER last_known_addr { host: "127.9.82.65" port: 46241 } } peers { permanent_uuid: "5d3db908b485446cbcbe6827b7bbbcd7" member_type: VOTER last_known_addr { host: "127.9.82.66" port: 37375 } } peers { permanent_uuid: "5289e655d9c647f499e5df5f236ce008" member_type: VOTER last_known_addr { host: "127.9.82.67" port: 36991 } }
07:01:10.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.162779 10314 tablet_replica.cc:439] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 11ae612cdfd94d638c0c35046ab8118d: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "11ae612cdfd94d638c0c35046ab8118d" member_type: VOTER last_known_addr { host: "127.9.82.65" port: 46241 } } peers { permanent_uuid: "5d3db908b485446cbcbe6827b7bbbcd7" member_type: VOTER last_known_addr { host: "127.9.82.66" port: 37375 } } peers { permanent_uuid: "5289e655d9c647f499e5df5f236ce008" member_type: VOTER last_known_addr { host: "127.9.82.67" port: 36991 } } }
07:01:10.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.163153 10314 tablet_replica.cc:442] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 11ae612cdfd94d638c0c35046ab8118d: This TxnStatusTablet replica's current role is: FOLLOWER
07:01:10.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.164450 10308 ts_tablet_manager.cc:1398] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 11ae612cdfd94d638c0c35046ab8118d: Time spent starting tablet: real 0.004s user 0.002s sys 0.001s
07:01:10.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.170951 10305 tablet_bootstrap.cc:492] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 5d3db908b485446cbcbe6827b7bbbcd7: No bootstrap required, opened a new log
07:01:10.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.171181 10305 ts_tablet_manager.cc:1367] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 5d3db908b485446cbcbe6827b7bbbcd7: Time spent bootstrapping tablet: real 0.021s user 0.012s sys 0.003s
07:01:10.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.172096 10305 raft_consensus.cc:348] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 5d3db908b485446cbcbe6827b7bbbcd7 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "11ae612cdfd94d638c0c35046ab8118d" member_type: VOTER last_known_addr { host: "127.9.82.65" port: 46241 } } peers { permanent_uuid: "5d3db908b485446cbcbe6827b7bbbcd7" member_type: VOTER last_known_addr { host: "127.9.82.66" port: 37375 } } peers { permanent_uuid: "5289e655d9c647f499e5df5f236ce008" member_type: VOTER last_known_addr { host: "127.9.82.67" port: 36991 } }
07:01:10.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.172380 10305 raft_consensus.cc:374] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 5d3db908b485446cbcbe6827b7bbbcd7 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:01:10.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.172464 10305 raft_consensus.cc:729] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 5d3db908b485446cbcbe6827b7bbbcd7 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5d3db908b485446cbcbe6827b7bbbcd7, State: Initialized, Role: FOLLOWER
07:01:10.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.172796 10305 consensus_queue.cc:260] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 5d3db908b485446cbcbe6827b7bbbcd7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "11ae612cdfd94d638c0c35046ab8118d" member_type: VOTER last_known_addr { host: "127.9.82.65" port: 46241 } } peers { permanent_uuid: "5d3db908b485446cbcbe6827b7bbbcd7" member_type: VOTER last_known_addr { host: "127.9.82.66" port: 37375 } } peers { permanent_uuid: "5289e655d9c647f499e5df5f236ce008" member_type: VOTER last_known_addr { host: "127.9.82.67" port: 36991 } }
07:01:10.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.173296 10311 tablet_replica.cc:439] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 5d3db908b485446cbcbe6827b7bbbcd7: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "11ae612cdfd94d638c0c35046ab8118d" member_type: VOTER last_known_addr { host: "127.9.82.65" port: 46241 } } peers { permanent_uuid: "5d3db908b485446cbcbe6827b7bbbcd7" member_type: VOTER last_known_addr { host: "127.9.82.66" port: 37375 } } peers { permanent_uuid: "5289e655d9c647f499e5df5f236ce008" member_type: VOTER last_known_addr { host: "127.9.82.67" port: 36991 } } }
07:01:10.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.173465 10305 ts_tablet_manager.cc:1398] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 5d3db908b485446cbcbe6827b7bbbcd7: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
07:01:10.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.173580 10311 tablet_replica.cc:442] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 5d3db908b485446cbcbe6827b7bbbcd7: This TxnStatusTablet replica's current role is: FOLLOWER
07:01:10.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.270896 10309 raft_consensus.cc:482] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 5289e655d9c647f499e5df5f236ce008 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:01:10.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.271189 10309 raft_consensus.cc:504] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 5289e655d9c647f499e5df5f236ce008 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "11ae612cdfd94d638c0c35046ab8118d" member_type: VOTER last_known_addr { host: "127.9.82.65" port: 46241 } } peers { permanent_uuid: "5d3db908b485446cbcbe6827b7bbbcd7" member_type: VOTER last_known_addr { host: "127.9.82.66" port: 37375 } } peers { permanent_uuid: "5289e655d9c647f499e5df5f236ce008" member_type: VOTER last_known_addr { host: "127.9.82.67" port: 36991 } }
07:01:10.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.272076 10309 leader_election.cc:288] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 5289e655d9c647f499e5df5f236ce008 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 11ae612cdfd94d638c0c35046ab8118d (127.9.82.65:46241), 5d3db908b485446cbcbe6827b7bbbcd7 (127.9.82.66:37375)
07:01:10.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.272739 10092 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fef62f88ed7a42d9b3f16fcaa8ac2250" candidate_uuid: "5289e655d9c647f499e5df5f236ce008" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5d3db908b485446cbcbe6827b7bbbcd7" is_pre_election: true
07:01:10.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.272765 9950 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fef62f88ed7a42d9b3f16fcaa8ac2250" candidate_uuid: "5289e655d9c647f499e5df5f236ce008" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "11ae612cdfd94d638c0c35046ab8118d" is_pre_election: true
07:01:10.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.272986 9950 raft_consensus.cc:2454] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 11ae612cdfd94d638c0c35046ab8118d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5289e655d9c647f499e5df5f236ce008 in term 0.
07:01:10.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.273041 10092 raft_consensus.cc:2454] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 5d3db908b485446cbcbe6827b7bbbcd7 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5289e655d9c647f499e5df5f236ce008 in term 0.
07:01:10.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.273475 10158 leader_election.cc:302] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 5289e655d9c647f499e5df5f236ce008 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 11ae612cdfd94d638c0c35046ab8118d, 5289e655d9c647f499e5df5f236ce008; no voters:
07:01:10.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.273916 10309 raft_consensus.cc:2781] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 5289e655d9c647f499e5df5f236ce008 [term 0 FOLLOWER]: Leader pre-election won for term 1
07:01:10.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.274020 10309 raft_consensus.cc:482] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 5289e655d9c647f499e5df5f236ce008 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
07:01:10.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.274108 10309 raft_consensus.cc:3037] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 5289e655d9c647f499e5df5f236ce008 [term 0 FOLLOWER]: Advancing to term 1
07:01:10.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.276800 10309 raft_consensus.cc:504] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 5289e655d9c647f499e5df5f236ce008 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "11ae612cdfd94d638c0c35046ab8118d" member_type: VOTER last_known_addr { host: "127.9.82.65" port: 46241 } } peers { permanent_uuid: "5d3db908b485446cbcbe6827b7bbbcd7" member_type: VOTER last_known_addr { host: "127.9.82.66" port: 37375 } } peers { permanent_uuid: "5289e655d9c647f499e5df5f236ce008" member_type: VOTER last_known_addr { host: "127.9.82.67" port: 36991 } }
07:01:10.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.277602 10309 leader_election.cc:288] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 5289e655d9c647f499e5df5f236ce008 [CANDIDATE]: Term 1 election: Requested vote from peers 11ae612cdfd94d638c0c35046ab8118d (127.9.82.65:46241), 5d3db908b485446cbcbe6827b7bbbcd7 (127.9.82.66:37375)
07:01:10.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.278189 9950 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fef62f88ed7a42d9b3f16fcaa8ac2250" candidate_uuid: "5289e655d9c647f499e5df5f236ce008" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "11ae612cdfd94d638c0c35046ab8118d"
07:01:10.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.278173 10092 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fef62f88ed7a42d9b3f16fcaa8ac2250" candidate_uuid: "5289e655d9c647f499e5df5f236ce008" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5d3db908b485446cbcbe6827b7bbbcd7"
07:01:10.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.278434 9950 raft_consensus.cc:3037] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 11ae612cdfd94d638c0c35046ab8118d [term 0 FOLLOWER]: Advancing to term 1
07:01:10.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.278455 10092 raft_consensus.cc:3037] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 5d3db908b485446cbcbe6827b7bbbcd7 [term 0 FOLLOWER]: Advancing to term 1
07:01:10.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.281522 9950 raft_consensus.cc:2454] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 11ae612cdfd94d638c0c35046ab8118d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5289e655d9c647f499e5df5f236ce008 in term 1.
07:01:10.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.281512 10092 raft_consensus.cc:2454] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 5d3db908b485446cbcbe6827b7bbbcd7 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5289e655d9c647f499e5df5f236ce008 in term 1.
07:01:10.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.282104 10157 leader_election.cc:302] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 5289e655d9c647f499e5df5f236ce008 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 3 yes votes; 0 no votes. yes voters: 11ae612cdfd94d638c0c35046ab8118d, 5289e655d9c647f499e5df5f236ce008, 5d3db908b485446cbcbe6827b7bbbcd7; no voters:
07:01:10.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.282500 10309 raft_consensus.cc:2781] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 5289e655d9c647f499e5df5f236ce008 [term 1 FOLLOWER]: Leader election won for term 1
07:01:10.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.282676 10309 raft_consensus.cc:686] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 5289e655d9c647f499e5df5f236ce008 [term 1 LEADER]: Becoming Leader. State: Replica: 5289e655d9c647f499e5df5f236ce008, State: Running, Role: LEADER
07:01:10.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.282946 10309 consensus_queue.cc:237] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 5289e655d9c647f499e5df5f236ce008 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "11ae612cdfd94d638c0c35046ab8118d" member_type: VOTER last_known_addr { host: "127.9.82.65" port: 46241 } } peers { permanent_uuid: "5d3db908b485446cbcbe6827b7bbbcd7" member_type: VOTER last_known_addr { host: "127.9.82.66" port: 37375 } } peers { permanent_uuid: "5289e655d9c647f499e5df5f236ce008" member_type: VOTER last_known_addr { host: "127.9.82.67" port: 36991 } }
07:01:10.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.285492 10320 tablet_replica.cc:439] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 5289e655d9c647f499e5df5f236ce008: TxnStatusTablet state changed. Reason: New leader 5289e655d9c647f499e5df5f236ce008. Latest consensus state: current_term: 1 leader_uuid: "5289e655d9c647f499e5df5f236ce008" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "11ae612cdfd94d638c0c35046ab8118d" member_type: VOTER last_known_addr { host: "127.9.82.65" port: 46241 } } peers { permanent_uuid: "5d3db908b485446cbcbe6827b7bbbcd7" member_type: VOTER last_known_addr { host: "127.9.82.66" port: 37375 } } peers { permanent_uuid: "5289e655d9c647f499e5df5f236ce008" member_type: VOTER last_known_addr { host: "127.9.82.67" port: 36991 } } }
07:01:10.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.285742 10320 tablet_replica.cc:442] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 5289e655d9c647f499e5df5f236ce008: This TxnStatusTablet replica's current role is: LEADER
07:01:10.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.286597 9587 catalog_manager.cc:5484] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 5289e655d9c647f499e5df5f236ce008 reported cstate change: term changed from 0 to 1, leader changed from <none> to 5289e655d9c647f499e5df5f236ce008 (127.9.82.67). New cstate: current_term: 1 leader_uuid: "5289e655d9c647f499e5df5f236ce008" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "11ae612cdfd94d638c0c35046ab8118d" member_type: VOTER last_known_addr { host: "127.9.82.65" port: 46241 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5d3db908b485446cbcbe6827b7bbbcd7" member_type: VOTER last_known_addr { host: "127.9.82.66" port: 37375 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5289e655d9c647f499e5df5f236ce008" member_type: VOTER last_known_addr { host: "127.9.82.67" port: 36991 } health_report { overall_health: HEALTHY } } }
07:01:10.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.296677 10335 txn_status_manager.cc:873] Waiting until node catch up with all replicated operations in previous term...
07:01:10.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.297044 10335 txn_status_manager.cc:929] Loading transaction status metadata into memory...
07:01:10.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.345005 9950 raft_consensus.cc:1261] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 11ae612cdfd94d638c0c35046ab8118d [term 1 FOLLOWER]: Refusing update from remote peer 5289e655d9c647f499e5df5f236ce008: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
07:01:10.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.345023 10092 raft_consensus.cc:1261] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 5d3db908b485446cbcbe6827b7bbbcd7 [term 1 FOLLOWER]: Refusing update from remote peer 5289e655d9c647f499e5df5f236ce008: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
07:01:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.346002 10309 consensus_queue.cc:1035] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 5289e655d9c647f499e5df5f236ce008 [LEADER]: Connected to new peer: Peer: permanent_uuid: "11ae612cdfd94d638c0c35046ab8118d" member_type: VOTER last_known_addr { host: "127.9.82.65" port: 46241 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
07:01:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.346396 10320 consensus_queue.cc:1035] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 5289e655d9c647f499e5df5f236ce008 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5d3db908b485446cbcbe6827b7bbbcd7" member_type: VOTER last_known_addr { host: "127.9.82.66" port: 37375 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
07:01:10.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.356009 10314 tablet_replica.cc:439] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 11ae612cdfd94d638c0c35046ab8118d: TxnStatusTablet state changed. Reason: New leader 5289e655d9c647f499e5df5f236ce008. Latest consensus state: current_term: 1 leader_uuid: "5289e655d9c647f499e5df5f236ce008" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "11ae612cdfd94d638c0c35046ab8118d" member_type: VOTER last_known_addr { host: "127.9.82.65" port: 46241 } } peers { permanent_uuid: "5d3db908b485446cbcbe6827b7bbbcd7" member_type: VOTER last_known_addr { host: "127.9.82.66" port: 37375 } } peers { permanent_uuid: "5289e655d9c647f499e5df5f236ce008" member_type: VOTER last_known_addr { host: "127.9.82.67" port: 36991 } } }
07:01:10.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.356246 10314 tablet_replica.cc:442] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 11ae612cdfd94d638c0c35046ab8118d: This TxnStatusTablet replica's current role is: FOLLOWER
07:01:10.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.361621 10338 mvcc.cc:205] Tried to move back new op lower bound from 7021014508922310656 to 7021014508682235904. Current Snapshot: MvccSnapshot[applied={T|T < 7021014508922310656}]
07:01:10.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.362350 10339 mvcc.cc:205] Tried to move back new op lower bound from 7021014508922310656 to 7021014508682235904. Current Snapshot: MvccSnapshot[applied={T|T < 7021014508922310656}]
07:01:10.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.364053 10314 tablet_replica.cc:439] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 11ae612cdfd94d638c0c35046ab8118d: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "5289e655d9c647f499e5df5f236ce008" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "11ae612cdfd94d638c0c35046ab8118d" member_type: VOTER last_known_addr { host: "127.9.82.65" port: 46241 } } peers { permanent_uuid: "5d3db908b485446cbcbe6827b7bbbcd7" member_type: VOTER last_known_addr { host: "127.9.82.66" port: 37375 } } peers { permanent_uuid: "5289e655d9c647f499e5df5f236ce008" member_type: VOTER last_known_addr { host: "127.9.82.67" port: 36991 } } }
07:01:10.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.364377 10314 tablet_replica.cc:442] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 11ae612cdfd94d638c0c35046ab8118d: This TxnStatusTablet replica's current role is: FOLLOWER
07:01:10.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.372953 10320 tablet_replica.cc:439] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 5289e655d9c647f499e5df5f236ce008: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "5289e655d9c647f499e5df5f236ce008" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "11ae612cdfd94d638c0c35046ab8118d" member_type: VOTER last_known_addr { host: "127.9.82.65" port: 46241 } } peers { permanent_uuid: "5d3db908b485446cbcbe6827b7bbbcd7" member_type: VOTER last_known_addr { host: "127.9.82.66" port: 37375 } } peers { permanent_uuid: "5289e655d9c647f499e5df5f236ce008" member_type: VOTER last_known_addr { host: "127.9.82.67" port: 36991 } } }
07:01:10.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.373267 10320 tablet_replica.cc:442] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 5289e655d9c647f499e5df5f236ce008: This TxnStatusTablet replica's current role is: LEADER
07:01:10.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.376935 10311 tablet_replica.cc:439] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 5d3db908b485446cbcbe6827b7bbbcd7: TxnStatusTablet state changed. Reason: New leader 5289e655d9c647f499e5df5f236ce008. Latest consensus state: current_term: 1 leader_uuid: "5289e655d9c647f499e5df5f236ce008" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "11ae612cdfd94d638c0c35046ab8118d" member_type: VOTER last_known_addr { host: "127.9.82.65" port: 46241 } } peers { permanent_uuid: "5d3db908b485446cbcbe6827b7bbbcd7" member_type: VOTER last_known_addr { host: "127.9.82.66" port: 37375 } } peers { permanent_uuid: "5289e655d9c647f499e5df5f236ce008" member_type: VOTER last_known_addr { host: "127.9.82.67" port: 36991 } } }
07:01:10.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.377307 10311 tablet_replica.cc:442] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 5d3db908b485446cbcbe6827b7bbbcd7: This TxnStatusTablet replica's current role is: FOLLOWER
07:01:10.381 [DEBUG - Time-limited test] (KuduTransaction.java:702) starting keepalive heartbeating with period 500 ms (txn ID 0)
07:01:10.385 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 8df132d56add4b74b452a3573f2f29b2
07:01:10.387 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 33dd825246c541e58f1e54d53dce8b17 for table 'txn_manager_fallback' with partition [0x00000000, 0x00000001)
07:01:10.388 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet db85828f290d4e92a96c56d6a71c1de6 for table 'txn_manager_fallback' with partition [0x00000001, 0x00000002)
07:01:10.388 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x00000000, upperBoundPartitionKey=0x00000001, ttl=299999, tablet=33dd825246c541e58f1e54d53dce8b17@[11ae612cdfd94d638c0c35046ab8118d(127.9.82.65:46241),5289e655d9c647f499e5df5f236ce008(127.9.82.67:36991)[L],5d3db908b485446cbcbe6827b7bbbcd7(127.9.82.66:37375)]}, Tablet{lowerBoundPartitionKey=0x00000001, upperBoundPartitionKey=0x00000002, ttl=299999, tablet=db85828f290d4e92a96c56d6a71c1de6@[11ae612cdfd94d638c0c35046ab8118d(127.9.82.65:46241),5289e655d9c647f499e5df5f236ce008(127.9.82.67:36991),5d3db908b485446cbcbe6827b7bbbcd7(127.9.82.66:37375)[L]]}, NonCoveredRange{lowerBoundPartitionKey=0x00000002, upperBoundPartitionKey=0x, ttl=299999}]
07:01:10.388 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=3), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.9.82.126:43671, [2ms] Sub RPC GetTableLocations: received response from server master-127.9.82.126:43671: OK, deferred=null) after lookup
07:01:10.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.391418 10340 mvcc.cc:205] Tried to move back new op lower bound from 7021014508922310656 to 7021014508682235904. Current Snapshot: MvccSnapshot[applied={T|T < 7021014508922310656}]
07:01:10.394 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 5289e655d9c647f499e5df5f236ce008(127.9.82.67:36991)] connecting to peer
07:01:10.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.398413 10345 tablet_replica.cc:439] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 5289e655d9c647f499e5df5f236ce008: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "5289e655d9c647f499e5df5f236ce008" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "11ae612cdfd94d638c0c35046ab8118d" member_type: VOTER last_known_addr { host: "127.9.82.65" port: 46241 } } peers { permanent_uuid: "5d3db908b485446cbcbe6827b7bbbcd7" member_type: VOTER last_known_addr { host: "127.9.82.66" port: 37375 } } peers { permanent_uuid: "5289e655d9c647f499e5df5f236ce008" member_type: VOTER last_known_addr { host: "127.9.82.67" port: 36991 } } }
07:01:10.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.398852 10345 tablet_replica.cc:442] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 5289e655d9c647f499e5df5f236ce008: This TxnStatusTablet replica's current role is: LEADER
07:01:10.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.399672 10311 tablet_replica.cc:439] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 5d3db908b485446cbcbe6827b7bbbcd7: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "5289e655d9c647f499e5df5f236ce008" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "11ae612cdfd94d638c0c35046ab8118d" member_type: VOTER last_known_addr { host: "127.9.82.65" port: 46241 } } peers { permanent_uuid: "5d3db908b485446cbcbe6827b7bbbcd7" member_type: VOTER last_known_addr { host: "127.9.82.66" port: 37375 } } peers { permanent_uuid: "5289e655d9c647f499e5df5f236ce008" member_type: VOTER last_known_addr { host: "127.9.82.67" port: 36991 } } }
07:01:10.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.399917 10311 tablet_replica.cc:442] T fef62f88ed7a42d9b3f16fcaa8ac2250 P 5d3db908b485446cbcbe6827b7bbbcd7: This TxnStatusTablet replica's current role is: FOLLOWER
07:01:10.409 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 5289e655d9c647f499e5df5f236ce008(127.9.82.67:36991)] Successfully connected to peer
07:01:10.430 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xaa2f2fb0, L:/127.0.0.1:58366 - R:/127.9.82.67:36991] using TOKEN/null
07:01:10.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.445257 10311 consensus_queue.cc:1035] T db85828f290d4e92a96c56d6a71c1de6 P 5d3db908b485446cbcbe6827b7bbbcd7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "11ae612cdfd94d638c0c35046ab8118d" member_type: VOTER last_known_addr { host: "127.9.82.65" port: 46241 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
07:01:10.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.461071 10311 consensus_queue.cc:1035] T db85828f290d4e92a96c56d6a71c1de6 P 5d3db908b485446cbcbe6827b7bbbcd7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5289e655d9c647f499e5df5f236ce008" member_type: VOTER last_known_addr { host: "127.9.82.67" port: 36991 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
07:01:10.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.470250 10091 raft_consensus.cc:1261] T 33dd825246c541e58f1e54d53dce8b17 P 5d3db908b485446cbcbe6827b7bbbcd7 [term 1 FOLLOWER]: Refusing update from remote peer 5289e655d9c647f499e5df5f236ce008: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
07:01:10.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.471197 10345 consensus_queue.cc:1035] T 33dd825246c541e58f1e54d53dce8b17 P 5289e655d9c647f499e5df5f236ce008 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5d3db908b485446cbcbe6827b7bbbcd7" member_type: VOTER last_known_addr { host: "127.9.82.66" port: 37375 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
07:01:10.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.472621 9949 raft_consensus.cc:1261] T 33dd825246c541e58f1e54d53dce8b17 P 11ae612cdfd94d638c0c35046ab8118d [term 1 FOLLOWER]: Refusing update from remote peer 5289e655d9c647f499e5df5f236ce008: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
07:01:10.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.476960 10345 consensus_queue.cc:1035] T 33dd825246c541e58f1e54d53dce8b17 P 5289e655d9c647f499e5df5f236ce008 [LEADER]: Connected to new peer: Peer: permanent_uuid: "11ae612cdfd94d638c0c35046ab8118d" member_type: VOTER last_known_addr { host: "127.9.82.65" port: 46241 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
07:01:10.517 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.9.82.126:43671
07:01:10.518 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
07:01:10.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.518800 9545 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu pid 9556
07:01:10.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.520484 9627 generic_service.cc:196] Checking for leaks (request via RPC)
07:01:10.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.648294 9545 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu with pid 9556
07:01:10.672 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.9.82.126:43671(127.9.82.126:43671)] handling channelInactive
07:01:10.673 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.82.126:43671(127.9.82.126:43671)] cleaning up while in state READY due to: connection closed
07:01:10.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:10.673889 9735 connection.cc:535] server connection from 127.0.0.1:38140 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
07:01:10.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:10.674610 9653 connection.cc:535] server connection from 127.0.0.1:50290 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
07:01:10.675 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:01:10.890 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.126:43671(127.9.82.126:43671)] connecting to peer
07:01:10.892 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:10.892 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.82.126:43671(127.9.82.126:43671)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:10.892 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location master-127.9.82.126:43671(127.9.82.126:43671) for tablet Kudu Master: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:10.908 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.126:43671(127.9.82.126:43671)] connecting to peer
07:01:10.909 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:10.909 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.82.126:43671(127.9.82.126:43671)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:10.909 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.126:43671: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:10.910 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345; will retry
07:01:10.928 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.126:43671(127.9.82.126:43671)] connecting to peer
07:01:10.929 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:10.929 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.82.126:43671(127.9.82.126:43671)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:10.929 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.126:43671: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:10.930 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345; will retry
07:01:10.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:10.943449 10138 heartbeater.cc:644] Failed to heartbeat to 127.9.82.126:43671 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.9.82.126:43671: connect: Connection refused (error 111)
07:01:10.947 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.126:43671(127.9.82.126:43671)] connecting to peer
07:01:10.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.947877 10357 raft_consensus.cc:482] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 9089901da6614cedb5be9c0f69cef850)
07:01:10.948 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:10.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.948293 10357 raft_consensus.cc:504] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9089901da6614cedb5be9c0f69cef850" member_type: VOTER last_known_addr { host: "127.9.82.126" port: 43671 } } peers { permanent_uuid: "7b5c12662b074d3e903fbcd05ed4a533" member_type: VOTER last_known_addr { host: "127.9.82.125" port: 45345 } } peers { permanent_uuid: "4a229e6d106c46caad88d051aa87c533" member_type: VOTER last_known_addr { host: "127.9.82.124" port: 38119 } }
07:01:10.948 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.82.126:43671(127.9.82.126:43671)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:10.948 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.126:43671: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:10.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.949926 10358 raft_consensus.cc:482] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 9089901da6614cedb5be9c0f69cef850)
07:01:10.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.950212 10357 leader_election.cc:288] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 9089901da6614cedb5be9c0f69cef850 (127.9.82.126:43671), 7b5c12662b074d3e903fbcd05ed4a533 (127.9.82.125:45345)
07:01:10.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.950706 10358 raft_consensus.cc:504] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9089901da6614cedb5be9c0f69cef850" member_type: VOTER last_known_addr { host: "127.9.82.126" port: 43671 } } peers { permanent_uuid: "7b5c12662b074d3e903fbcd05ed4a533" member_type: VOTER last_known_addr { host: "127.9.82.125" port: 45345 } } peers { permanent_uuid: "4a229e6d106c46caad88d051aa87c533" member_type: VOTER last_known_addr { host: "127.9.82.124" port: 38119 } }
07:01:10.951 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345; will retry
07:01:10.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.951053 9678 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4a229e6d106c46caad88d051aa87c533" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "7b5c12662b074d3e903fbcd05ed4a533" is_pre_election: true
07:01:10.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.951342 9678 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4a229e6d106c46caad88d051aa87c533 in term 1.
07:01:10.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.951968 9733 leader_election.cc:302] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4a229e6d106c46caad88d051aa87c533, 7b5c12662b074d3e903fbcd05ed4a533; no voters:
07:01:10.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.952055 9759 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7b5c12662b074d3e903fbcd05ed4a533" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "4a229e6d106c46caad88d051aa87c533" is_pre_election: true
07:01:10.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.952250 10358 leader_election.cc:288] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 9089901da6614cedb5be9c0f69cef850 (127.9.82.126:43671), 4a229e6d106c46caad88d051aa87c533 (127.9.82.124:38119)
07:01:10.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.952284 9759 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7b5c12662b074d3e903fbcd05ed4a533 in term 1.
07:01:10.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:10.952018 9736 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.82.126:43671: connect: Connection refused (error 111)
07:01:10.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.952618 10357 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533 [term 1 FOLLOWER]: Leader pre-election won for term 2
07:01:10.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.952704 10357 raft_consensus.cc:482] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 9089901da6614cedb5be9c0f69cef850)
07:01:10.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.952766 10357 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533 [term 1 FOLLOWER]: Advancing to term 2
07:01:10.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.952858 9654 leader_election.cc:302] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4a229e6d106c46caad88d051aa87c533, 7b5c12662b074d3e903fbcd05ed4a533; no voters:
07:01:10.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.953203 10358 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 1 FOLLOWER]: Leader pre-election won for term 2
07:01:10.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.953447 10358 raft_consensus.cc:482] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 9089901da6614cedb5be9c0f69cef850)
07:01:10.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.953681 10358 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 1 FOLLOWER]: Advancing to term 2
07:01:10.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.956290 10357 raft_consensus.cc:504] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9089901da6614cedb5be9c0f69cef850" member_type: VOTER last_known_addr { host: "127.9.82.126" port: 43671 } } peers { permanent_uuid: "7b5c12662b074d3e903fbcd05ed4a533" member_type: VOTER last_known_addr { host: "127.9.82.125" port: 45345 } } peers { permanent_uuid: "4a229e6d106c46caad88d051aa87c533" member_type: VOTER last_known_addr { host: "127.9.82.124" port: 38119 } }
07:01:10.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.957634 10358 raft_consensus.cc:504] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9089901da6614cedb5be9c0f69cef850" member_type: VOTER last_known_addr { host: "127.9.82.126" port: 43671 } } peers { permanent_uuid: "7b5c12662b074d3e903fbcd05ed4a533" member_type: VOTER last_known_addr { host: "127.9.82.125" port: 45345 } } peers { permanent_uuid: "4a229e6d106c46caad88d051aa87c533" member_type: VOTER last_known_addr { host: "127.9.82.124" port: 38119 } }
07:01:10.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.958360 9678 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4a229e6d106c46caad88d051aa87c533" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "7b5c12662b074d3e903fbcd05ed4a533"
07:01:10.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.958819 9678 raft_consensus.cc:2379] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 4a229e6d106c46caad88d051aa87c533 in current term 2: Already voted for candidate 7b5c12662b074d3e903fbcd05ed4a533 in this term.
07:01:10.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.959229 10358 leader_election.cc:288] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [CANDIDATE]: Term 2 election: Requested vote from peers 9089901da6614cedb5be9c0f69cef850 (127.9.82.126:43671), 4a229e6d106c46caad88d051aa87c533 (127.9.82.124:38119)
07:01:10.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.959411 9759 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7b5c12662b074d3e903fbcd05ed4a533" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "4a229e6d106c46caad88d051aa87c533"
07:01:10.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.959779 9759 raft_consensus.cc:2379] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 7b5c12662b074d3e903fbcd05ed4a533 in current term 2: Already voted for candidate 4a229e6d106c46caad88d051aa87c533 in this term.
07:01:10.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.960778 10357 leader_election.cc:288] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533 [CANDIDATE]: Term 2 election: Requested vote from peers 9089901da6614cedb5be9c0f69cef850 (127.9.82.126:43671), 7b5c12662b074d3e903fbcd05ed4a533 (127.9.82.125:45345)
07:01:10.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:10.962239 9736 leader_election.cc:334] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 9089901da6614cedb5be9c0f69cef850 (127.9.82.126:43671): Network error: Client connection negotiation failed: client connection to 127.9.82.126:43671: connect: Connection refused (error 111)
07:01:10.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:10.962939 9655 leader_election.cc:334] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 9089901da6614cedb5be9c0f69cef850 (127.9.82.126:43671): Network error: Client connection negotiation failed: client connection to 127.9.82.126:43671: connect: Connection refused (error 111)
07:01:10.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:10.963181 9655 leader_election.cc:334] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 9089901da6614cedb5be9c0f69cef850 (127.9.82.126:43671): Network error: Client connection negotiation failed: client connection to 127.9.82.126:43671: connect: Connection refused (error 111)
07:01:10.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.963295 9655 leader_election.cc:302] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 7b5c12662b074d3e903fbcd05ed4a533; no voters: 4a229e6d106c46caad88d051aa87c533, 9089901da6614cedb5be9c0f69cef850
07:01:10.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:10.963618 9736 leader_election.cc:334] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 9089901da6614cedb5be9c0f69cef850 (127.9.82.126:43671): Network error: Client connection negotiation failed: client connection to 127.9.82.126:43671: connect: Connection refused (error 111)
07:01:10.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.963661 10358 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
07:01:10.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.963724 9736 leader_election.cc:302] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4a229e6d106c46caad88d051aa87c533; no voters: 7b5c12662b074d3e903fbcd05ed4a533, 9089901da6614cedb5be9c0f69cef850
07:01:10.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:10.963989 10357 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
07:01:10.967 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.126:43671(127.9.82.126:43671)] connecting to peer
07:01:10.968 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:10.968 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.82.126:43671(127.9.82.126:43671)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:10.968 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.126:43671: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:10.969 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345; will retry
07:01:10.987 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.126:43671(127.9.82.126:43671)] connecting to peer
07:01:10.988 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:10.988 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.82.126:43671(127.9.82.126:43671)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:10.989 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.126:43671: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:10.989 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345; will retry
07:01:11.067 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.126:43671(127.9.82.126:43671)] connecting to peer
07:01:11.069 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:11.069 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.82.126:43671(127.9.82.126:43671)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:11.069 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.126:43671: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:11.070 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345; will retry
07:01:11.108 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.126:43671(127.9.82.126:43671)] connecting to peer
07:01:11.109 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:11.109 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.82.126:43671(127.9.82.126:43671)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:11.109 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.126:43671: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:11.110 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345; will retry
07:01:11.111 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=223), Traces: [0ms] sending RPC to server master-127.9.82.126:43671, [5ms] received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [5ms] delaying RPC due to: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [22ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [23ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [23ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [42ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [43ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [43ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [61ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [62ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [64ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [81ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [82ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [101ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [102ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [102ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [182ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [183ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [183ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [220ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [222ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [223ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: OK, [223ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, deferred=Deferred@208746467(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$82/255253488@4ebf62aa -> (continuation of Deferred@1150472448 after retry RPC after error@1659898599) -> (continuation of Deferred@1731979125 after retry RPC after error@1386509066) -> (continuation of Deferred@7123599 after retry RPC after error@1646868311) -> (continuation of Deferred@1519481896 after retry RPC after error@735866963) -> (continuation of Deferred@1321293677 after retry RPC after error@1702445677) -> (continuation of Deferred@874007262 after retry RPC after error@2081356518), errback=org.apache.kudu.client.KuduTransaction$$Lambda$83/851091760@3830b035 -> (continuation of Deferred@1150472448 after retry RPC after error@1659898599) -> (continuation of Deferred@1731979125 after retry RPC after error@1386509066) -> (continuation of Deferred@7123599 after retry RPC after error@1646868311) -> (continuation of Deferred@1519481896 after retry RPC after error@735866963) -> (continuation of Deferred@1321293677 after retry RPC after error@1702445677) -> (continuation of Deferred@874007262 after retry RPC after error@2081356518)))
07:01:11.112 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=223), Traces: [0ms] sending RPC to server master-127.9.82.126:43671, [5ms] received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [5ms] delaying RPC due to: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [22ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [23ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [23ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [42ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [43ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [43ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [61ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [62ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [64ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [81ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [82ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [101ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [102ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [102ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [182ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [183ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [183ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [220ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [222ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [223ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: OK, [223ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, deferred=Deferred@208746467(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$82/255253488@4ebf62aa -> (continuation of Deferred@1150472448 after retry RPC after error@1659898599) -> (continuation of Deferred@1731979125 after retry RPC after error@1386509066) -> (continuation of Deferred@7123599 after retry RPC after error@1646868311) -> (continuation of Deferred@1519481896 after retry RPC after error@735866963) -> (continuation of Deferred@1321293677 after retry RPC after error@1702445677) -> (continuation of Deferred@874007262 after retry RPC after error@2081356518), errback=org.apache.kudu.client.KuduTransaction$$Lambda$83/851091760@3830b035 -> (continuation of Deferred@1150472448 after retry RPC after error@1659898599) -> (continuation of Deferred@1731979125 after retry RPC after error@1386509066) -> (continuation of Deferred@7123599 after retry RPC after error@1646868311) -> (continuation of Deferred@1519481896 after retry RPC after error@735866963) -> (continuation of Deferred@1321293677 after retry RPC after error@1702445677) -> (continuation of Deferred@874007262 after retry RPC after error@2081356518)))
07:01:11.127 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.126:43671(127.9.82.126:43671)] connecting to peer
07:01:11.128 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:11.129 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.82.126:43671(127.9.82.126:43671)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:11.129 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.126:43671: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:11.129 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345; will retry
07:01:11.147 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.126:43671(127.9.82.126:43671)] connecting to peer
07:01:11.149 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:11.149 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.82.126:43671(127.9.82.126:43671)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:11.149 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.126:43671: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:11.149 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345; will retry
07:01:11.167 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.126:43671(127.9.82.126:43671)] connecting to peer
07:01:11.168 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:11.168 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.82.126:43671(127.9.82.126:43671)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:11.168 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.126:43671: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:11.169 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345; will retry
07:01:11.187 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.126:43671(127.9.82.126:43671)] connecting to peer
07:01:11.189 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:11.189 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.82.126:43671(127.9.82.126:43671)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:11.189 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.126:43671: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:11.190 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345; will retry
07:01:11.207 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.126:43671(127.9.82.126:43671)] connecting to peer
07:01:11.208 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:11.208 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.82.126:43671(127.9.82.126:43671)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:11.209 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.126:43671: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:11.210 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345; will retry
07:01:11.227 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.126:43671(127.9.82.126:43671)] connecting to peer
07:01:11.228 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:11.228 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.82.126:43671(127.9.82.126:43671)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:11.229 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.126:43671: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:11.230 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345; will retry
07:01:11.287 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.126:43671(127.9.82.126:43671)] connecting to peer
07:01:11.289 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:11.289 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.82.126:43671(127.9.82.126:43671)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:11.289 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.126:43671: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:11.290 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345; will retry
07:01:11.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:11.330852 10357 raft_consensus.cc:482] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:01:11.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:11.331138 10357 raft_consensus.cc:504] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9089901da6614cedb5be9c0f69cef850" member_type: VOTER last_known_addr { host: "127.9.82.126" port: 43671 } } peers { permanent_uuid: "7b5c12662b074d3e903fbcd05ed4a533" member_type: VOTER last_known_addr { host: "127.9.82.125" port: 45345 } } peers { permanent_uuid: "4a229e6d106c46caad88d051aa87c533" member_type: VOTER last_known_addr { host: "127.9.82.124" port: 38119 } }
07:01:11.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:11.332038 10357 leader_election.cc:288] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 9089901da6614cedb5be9c0f69cef850 (127.9.82.126:43671), 7b5c12662b074d3e903fbcd05ed4a533 (127.9.82.125:45345)
07:01:11.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:11.332770 9678 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4a229e6d106c46caad88d051aa87c533" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "7b5c12662b074d3e903fbcd05ed4a533" is_pre_election: true
07:01:11.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:11.333142 9678 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4a229e6d106c46caad88d051aa87c533 in term 2.
07:01:11.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:11.333840 9733 leader_election.cc:302] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4a229e6d106c46caad88d051aa87c533, 7b5c12662b074d3e903fbcd05ed4a533; no voters:
07:01:11.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:11.334224 10357 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533 [term 2 FOLLOWER]: Leader pre-election won for term 3
07:01:11.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:11.334214 9736 leader_election.cc:334] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 9089901da6614cedb5be9c0f69cef850 (127.9.82.126:43671): Network error: Client connection negotiation failed: client connection to 127.9.82.126:43671: connect: Connection refused (error 111)
07:01:11.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:11.334359 10357 raft_consensus.cc:482] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
07:01:11.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:11.334465 10357 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533 [term 2 FOLLOWER]: Advancing to term 3
07:01:11.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:11.336506 10357 raft_consensus.cc:504] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9089901da6614cedb5be9c0f69cef850" member_type: VOTER last_known_addr { host: "127.9.82.126" port: 43671 } } peers { permanent_uuid: "7b5c12662b074d3e903fbcd05ed4a533" member_type: VOTER last_known_addr { host: "127.9.82.125" port: 45345 } } peers { permanent_uuid: "4a229e6d106c46caad88d051aa87c533" member_type: VOTER last_known_addr { host: "127.9.82.124" port: 38119 } }
07:01:11.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:11.337244 10357 leader_election.cc:288] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533 [CANDIDATE]: Term 3 election: Requested vote from peers 9089901da6614cedb5be9c0f69cef850 (127.9.82.126:43671), 7b5c12662b074d3e903fbcd05ed4a533 (127.9.82.125:45345)
07:01:11.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:11.337829 9678 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4a229e6d106c46caad88d051aa87c533" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "7b5c12662b074d3e903fbcd05ed4a533"
07:01:11.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:11.338101 9678 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 2 FOLLOWER]: Advancing to term 3
07:01:11.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:11.338631 9736 leader_election.cc:334] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 9089901da6614cedb5be9c0f69cef850 (127.9.82.126:43671): Network error: Client connection negotiation failed: client connection to 127.9.82.126:43671: connect: Connection refused (error 111)
07:01:11.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:11.340387 9678 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4a229e6d106c46caad88d051aa87c533 in term 3.
07:01:11.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:11.341058 9733 leader_election.cc:302] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 4a229e6d106c46caad88d051aa87c533, 7b5c12662b074d3e903fbcd05ed4a533; no voters: 9089901da6614cedb5be9c0f69cef850
07:01:11.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:11.341460 10357 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533 [term 3 FOLLOWER]: Leader election won for term 3
07:01:11.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:11.341904 10357 raft_consensus.cc:686] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533 [term 3 LEADER]: Becoming Leader. State: Replica: 4a229e6d106c46caad88d051aa87c533, State: Running, Role: LEADER
07:01:11.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:11.342481 10357 consensus_queue.cc:237] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9089901da6614cedb5be9c0f69cef850" member_type: VOTER last_known_addr { host: "127.9.82.126" port: 43671 } } peers { permanent_uuid: "7b5c12662b074d3e903fbcd05ed4a533" member_type: VOTER last_known_addr { host: "127.9.82.125" port: 45345 } } peers { permanent_uuid: "4a229e6d106c46caad88d051aa87c533" member_type: VOTER last_known_addr { host: "127.9.82.124" port: 38119 } }
07:01:11.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:11.344604 10363 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4a229e6d106c46caad88d051aa87c533. Latest consensus state: current_term: 3 leader_uuid: "4a229e6d106c46caad88d051aa87c533" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9089901da6614cedb5be9c0f69cef850" member_type: VOTER last_known_addr { host: "127.9.82.126" port: 43671 } } peers { permanent_uuid: "7b5c12662b074d3e903fbcd05ed4a533" member_type: VOTER last_known_addr { host: "127.9.82.125" port: 45345 } } peers { permanent_uuid: "4a229e6d106c46caad88d051aa87c533" member_type: VOTER last_known_addr { host: "127.9.82.124" port: 38119 } } }
07:01:11.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:11.345000 10363 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533 [sys.catalog]: This master's current role is: LEADER
07:01:11.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:11.345754 10365 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
07:01:11.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:11.349023 10365 catalog_manager.cc:669] Loaded metadata for table txn_manager_fallback [id=8df132d56add4b74b452a3573f2f29b2]
07:01:11.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:11.349582 10365 catalog_manager.cc:669] Loaded metadata for table kudu_system.kudu_transactions [id=c7e9c9d4914946a2b4e4b9116312e2cb]
07:01:11.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:11.351713 10365 tablet_loader.cc:96] loaded metadata for tablet 33dd825246c541e58f1e54d53dce8b17 (table txn_manager_fallback [id=8df132d56add4b74b452a3573f2f29b2])
07:01:11.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:11.352193 10365 tablet_loader.cc:96] loaded metadata for tablet db85828f290d4e92a96c56d6a71c1de6 (table txn_manager_fallback [id=8df132d56add4b74b452a3573f2f29b2])
07:01:11.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:11.352489 10365 tablet_loader.cc:96] loaded metadata for tablet fef62f88ed7a42d9b3f16fcaa8ac2250 (table kudu_system.kudu_transactions [id=c7e9c9d4914946a2b4e4b9116312e2cb])
07:01:11.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:11.352752 10365 catalog_manager.cc:1484] Initializing Kudu cluster ID...
07:01:11.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:11.353832 10365 catalog_manager.cc:1259] Loaded cluster ID: 38c132b220504f5c90d8ff5a01d0e540
07:01:11.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:11.353930 10365 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
07:01:11.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:11.355384 10365 catalog_manager.cc:1504] Loading token signing keys...
07:01:11.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:11.356374 10365 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533: Loaded TSK: 0
07:01:11.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:11.357517 10365 catalog_manager.cc:1514] Initializing in-progress tserver states...
07:01:11.367 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.126:43671(127.9.82.126:43671)] connecting to peer
07:01:11.368 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:11.368 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.82.126:43671(127.9.82.126:43671)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:11.368 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.126:43671: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:11.371 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
07:01:11.372 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=245), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [3ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [3ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: OK, [3ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [23ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [23ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [23ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [42ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [43ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [43ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [63ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [63ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [82ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [83ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: OK, [84ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [102ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [103ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: OK, [104ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [104ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [163ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [164ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [164ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: OK, [164ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [240ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [242ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [244ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [245ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: OK, deferred=Deferred@2066162893(state=PENDING, result=null, callback=(continuation of Deferred@411151327 after retry RPC after error@1756881438) -> (continuation of Deferred@974220168 after retry RPC after error@2018648395) -> (continuation of Deferred@1838719693 after retry RPC after error@988289650) -> (continuation of Deferred@185887628 after retry RPC after error@1655000028) -> (continuation of Deferred@1250104207 after retry RPC after error@809675559) -> (continuation of Deferred@413335208 after retry RPC after error@1351724491) -> (continuation of Deferred@48022333 after retry RPC after error@660757458), errback=(continuation of Deferred@411151327 after retry RPC after error@1756881438) -> (continuation of Deferred@974220168 after retry RPC after error@2018648395) -> (continuation of Deferred@1838719693 after retry RPC after error@988289650) -> (continuation of Deferred@185887628 after retry RPC after error@1655000028) -> (continuation of Deferred@1250104207 after retry RPC after error@809675559) -> (continuation of Deferred@413335208 after retry RPC after error@1351724491) -> (continuation of Deferred@48022333 after retry RPC after error@660757458))) after lookup
07:01:11.387 [DEBUG - pool-46-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=9, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [3ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [3ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: OK, [3ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [23ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [23ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [23ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [42ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [43ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [43ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [63ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [63ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [82ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [83ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: OK, [84ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [102ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [103ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: OK, [104ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [104ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [163ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [164ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [164ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: OK, [164ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [240ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [242ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [244ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [245ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: OK, [246ms] sending RPC to server master-127.9.82.124:38119, deferred=Deferred@2066162893(state=PENDING, result=null, callback=(continuation of Deferred@411151327 after retry RPC after error@1756881438) -> (continuation of Deferred@974220168 after retry RPC after error@2018648395) -> (continuation of Deferred@1838719693 after retry RPC after error@988289650) -> (continuation of Deferred@185887628 after retry RPC after error@1655000028) -> (continuation of Deferred@1250104207 after retry RPC after error@809675559) -> (continuation of Deferred@413335208 after retry RPC after error@1351724491) -> (continuation of Deferred@48022333 after retry RPC after error@660757458) -> (continuation of Deferred@1954605781 after retry RPC@983801121), errback=(continuation of Deferred@411151327 after retry RPC after error@1756881438) -> (continuation of Deferred@974220168 after retry RPC after error@2018648395) -> (continuation of Deferred@1838719693 after retry RPC after error@988289650) -> (continuation of Deferred@185887628 after retry RPC after error@1655000028) -> (continuation of Deferred@1250104207 after retry RPC after error@809675559) -> (continuation of Deferred@413335208 after retry RPC after error@1351724491) -> (continuation of Deferred@48022333 after retry RPC after error@660757458) -> (continuation of Deferred@1954605781 after retry RPC@983801121)))
07:01:11.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:11.403810 9779 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:38182 (request call id 16) took 30 ms (client timeout 4 ms). Trace:
07:01:11.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:11.404182 9779 rpcz_store.cc:269] 0426 07:01:11.373586 (+ 0us) service_pool.cc:168] Inserting onto call queue
07:01:11.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 07:01:11.373672 (+ 86us) service_pool.cc:225] Handling call
07:01:11.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 07:01:11.403690 (+ 30018us) inbound_call.cc:173] Queueing success response
07:01:11.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"init.queue_time_us":496,"thread_start_us":1753,"threads_started":3}
07:01:11.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/util/monotime.cc:332:28: runtime error: signed integer overflow: 1349017026019 + 9223370687837750809 cannot be represented in type 'long'
07:01:11.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:11.448252 9736 consensus_peers.cc:485] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533 -> Peer 9089901da6614cedb5be9c0f69cef850 (127.9.82.126:43671): Couldn't send request to peer 9089901da6614cedb5be9c0f69cef850. Status: Network error: Client connection negotiation failed: client connection to 127.9.82.126:43671: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
07:01:11.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:11.450804 9678 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 3 FOLLOWER]: Refusing update from remote peer 4a229e6d106c46caad88d051aa87c533: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 3 index: 12. (index mismatch)
07:01:11.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:11.451927 10363 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7b5c12662b074d3e903fbcd05ed4a533" member_type: VOTER last_known_addr { host: "127.9.82.125" port: 45345 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
07:01:11.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:11.454614 10358 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4a229e6d106c46caad88d051aa87c533. Latest consensus state: current_term: 3 leader_uuid: "4a229e6d106c46caad88d051aa87c533" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9089901da6614cedb5be9c0f69cef850" member_type: VOTER last_known_addr { host: "127.9.82.126" port: 43671 } } peers { permanent_uuid: "7b5c12662b074d3e903fbcd05ed4a533" member_type: VOTER last_known_addr { host: "127.9.82.125" port: 45345 } } peers { permanent_uuid: "4a229e6d106c46caad88d051aa87c533" member_type: VOTER last_known_addr { host: "127.9.82.124" port: 38119 } } }
07:01:11.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:11.454974 10358 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [sys.catalog]: This master's current role is: FOLLOWER
07:01:11.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:11.466969 10162 txn_status_manager.cc:1390] automatically aborted stale txn (ID 0) past 1.029s from last keepalive heartbeat (effective timeout is 1.000s)
07:01:11.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:11.476315 10209 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.9.82.126:43671)
07:01:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:11.493476 10386 tablet_replica.cc:1298] Aborted: operation has been aborted: cancelling pending write operations
07:01:11.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:11.500747 10358 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "4a229e6d106c46caad88d051aa87c533" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9089901da6614cedb5be9c0f69cef850" member_type: VOTER last_known_addr { host: "127.9.82.126" port: 43671 } } peers { permanent_uuid: "7b5c12662b074d3e903fbcd05ed4a533" member_type: VOTER last_known_addr { host: "127.9.82.125" port: 45345 } } peers { permanent_uuid: "4a229e6d106c46caad88d051aa87c533" member_type: VOTER last_known_addr { host: "127.9.82.124" port: 38119 } } }
07:01:11.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:11.501137 10358 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [sys.catalog]: This master's current role is: FOLLOWER
07:01:11.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:11.515894 9749 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:38206:
07:01:11.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
07:01:11.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
07:01:11.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:01:11.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
07:01:11.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
07:01:11.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
07:01:11.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:01:11.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
07:01:11.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
07:01:11.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:01:11.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:01:11.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:01:11.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:01:11.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
07:01:11.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
07:01:11.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
07:01:11.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:01:11.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
07:01:11.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
07:01:11.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:01:11.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:01:11.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:01:11.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:01:11.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
07:01:11.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
07:01:11.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
07:01:11.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:01:11.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
07:01:11.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
07:01:11.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:01:11.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:01:11.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:01:11.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:01:11.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
07:01:11.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
07:01:11.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
07:01:11.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
07:01:11.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
07:01:11.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
07:01:11.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
07:01:11.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
07:01:11.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:01:11.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:01:11.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
07:01:11.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
07:01:11.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
07:01:11.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
07:01:11.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:01:11.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
07:01:11.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
07:01:11.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
07:01:11.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
07:01:11.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:01:11.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:01:11.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
07:01:11.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
07:01:11.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:11.525552 10363 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "4a229e6d106c46caad88d051aa87c533" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9089901da6614cedb5be9c0f69cef850" member_type: VOTER last_known_addr { host: "127.9.82.126" port: 43671 } } peers { permanent_uuid: "7b5c12662b074d3e903fbcd05ed4a533" member_type: VOTER last_known_addr { host: "127.9.82.125" port: 45345 } } peers { permanent_uuid: "4a229e6d106c46caad88d051aa87c533" member_type: VOTER last_known_addr { host: "127.9.82.124" port: 38119 } } }
07:01:11.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:11.525898 10363 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533 [sys.catalog]: This master's current role is: LEADER
07:01:11.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:11.529073 9749 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
07:01:11.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:11.552465 9996 heartbeater.cc:644] Failed to heartbeat to 127.9.82.126:43671 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.9.82.126:43671: connect: Connection refused (error 111)
07:01:11.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:11.582144 10279 heartbeater.cc:644] Failed to heartbeat to 127.9.82.126:43671 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.9.82.126:43671: connect: Connection refused (error 111)
07:01:11.599 [DEBUG - kudu-nio-1] (KuduTransaction.java:799) terminating keepalive task (txn ID 0) due to exception org.apache.kudu.client.NonRecoverableException: Failed to write to server: 5289e655d9c647f499e5df5f236ce008 (127.9.82.67:36991): transaction ID 0 is not available for further commits: state: ABORTED user: "slave" start_timestamp: 1714114870 last_transition_timestamp: 1714114871
07:01:11.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:11.978811 9736 consensus_peers.cc:485] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533 -> Peer 9089901da6614cedb5be9c0f69cef850 (127.9.82.126:43671): Couldn't send request to peer 9089901da6614cedb5be9c0f69cef850. Status: Network error: Client connection negotiation failed: client connection to 127.9.82.126:43671: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
07:01:12.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 0x7f672eba7f98 in kudu::operator+(kudu::MonoTime const&, kudu::MonoDelta const&) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/util/monotime.cc:332:28
07:01:12.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 0xa4f4b5 in kudu::CountDownLatch::WaitFor(kudu::MonoDelta const&) const /home/jenkins-slave/workspace/build_and_test@3/src/kudu/util/countdown_latch.h:97:38
07:01:12.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 0x7f67421b6bb2 in kudu::Promise<kudu::Status>::WaitFor(kudu::MonoDelta const&) const /home/jenkins-slave/workspace/build_and_test@3/src/kudu/util/promise.h:57:16
07:01:12.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 0x7f67421a7384 in kudu::master::Master::WaitForTxnManagerInit(kudu::MonoTime) const /home/jenkins-slave/workspace/build_and_test@3/src/kudu/master/master.cc:458:48
07:01:12.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:12.518987 9736 consensus_peers.cc:485] T 00000000000000000000000000000000 P 4a229e6d106c46caad88d051aa87c533 -> Peer 9089901da6614cedb5be9c0f69cef850 (127.9.82.126:43671): Couldn't send request to peer 9089901da6614cedb5be9c0f69cef850. Status: Network error: Client connection negotiation failed: client connection to 127.9.82.126:43671: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
07:01:12.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 0x7f67423008ee in kudu::transactions::TxnManager::CheckInitialized(kudu::MonoTime const&) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/master/txn_manager.cc:283:21
07:01:12.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 0x7f67423014cb in kudu::transactions::TxnManager::KeepTransactionAlive(long, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, kudu::MonoTime const&) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/master/txn_manager.cc:226:3
07:01:12.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:12.524582 9995 heartbeater.cc:503] Master 127.9.82.124:38119 was elected leader, sending a full tablet report...
07:01:12.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:12.525504 10278 heartbeater.cc:503] Master 127.9.82.124:38119 was elected leader, sending a full tablet report...
07:01:12.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 0x7f674230613f in kudu::transactions::TxnManagerServiceImpl::KeepTransactionAlive(kudu::transactions::KeepTransactionAliveRequestPB const*, kudu::transactions::KeepTransactionAliveResponsePB*, kudu::rpc::RpcContext*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/master/txn_manager_service.cc:159:42
07:01:12.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 0x7f673c188546 in kudu::transactions::TxnManagerServiceIf::TxnManagerServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_9::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /home/jenkins-slave/workspace/build_and_test@3/build/asan/src/kudu/master/txn_manager.service.cc:271:13
07:01:12.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 0x7f673c188275 in std::_Function_handler<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*), kudu::transactions::TxnManagerServiceIf::TxnManagerServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_9>::_M_invoke(std::_Any_data const&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) ../../../include/c++/7.5.0/bits/std_function.h:316:2
07:01:12.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 0x7f673594b4fe in std::function<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const ../../../include/c++/7.5.0/bits/std_function.h:706:14
07:01:12.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 0x7f673594a3de in kudu::rpc::GeneratedServiceIf::Handle(kudu::rpc::InboundCall*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/rpc/service_if.cc:137:3
07:01:12.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 0x7f67359508b5 in kudu::rpc::ServicePool::RunThread() /home/jenkins-slave/workspace/build_and_test@3/src/kudu/rpc/service_pool.cc:229:15
07:01:12.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 0x7f67359511df in kudu::rpc::ServicePool::Init(int)::$_0::operator()() const /home/jenkins-slave/workspace/build_and_test@3/src/kudu/rpc/service_pool.cc:92:5
07:01:12.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 0x7f6735951008 in std::_Function_handler<void (), kudu::rpc::ServicePool::Init(int)::$_0>::_M_invoke(std::_Any_data const&) ../../../include/c++/7.5.0/bits/std_function.h:316:2
07:01:12.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 0xa6022c in std::function<void ()>::operator()() const ../../../include/c++/7.5.0/bits/std_function.h:706:14
07:01:12.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 0x7f672ec6cc1d in kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test@3/src/kudu/util/thread.cc:694:3
07:01:12.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 0x7f6731fef6da in start_thread (/lib/x86_64-linux-gnu/libpthread.so.0+0x76da)
07:01:12.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 0x7f672ae3171e in clone (/lib/x86_64-linux-gnu/libc.so.6+0x12171e)
07:01:12.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:01:12.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) SUMMARY: UndefinedBehaviorSanitizer: undefined-behavior /home/jenkins-slave/workspace/build_and_test@3/src/kudu/util/monotime.cc:332:28 in
07:01:12.580 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.9.82.124:38119(127.9.82.124:38119)] handling channelInactive
07:01:12.581 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.82.124:38119(127.9.82.124:38119)] cleaning up while in state READY due to: connection closed
07:01:12.581 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location master-127.9.82.124:38119(127.9.82.124:38119) for tablet Kudu Master: connection closed
07:01:12.587 [WARN - pool-46-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
at java.lang.Thread.run(Thread.java:748)
07:01:12.675 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.9.82.126:43671(127.9.82.126:43671)] connecting to peer
07:01:12.676 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.9.82.124:38119(127.9.82.124:38119)] connecting to peer
07:01:12.676 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:12.676 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.82.126:43671(127.9.82.126:43671)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:12.676 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:12.676 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.126:43671: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:12.676 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.82.124:38119(127.9.82.124:38119)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:12.677 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.124:38119: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:12.678 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345; will retry
07:01:12.687 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.126:43671(127.9.82.126:43671)] connecting to peer
07:01:12.688 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.124:38119(127.9.82.124:38119)] connecting to peer
07:01:12.688 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:12.688 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.82.126:43671(127.9.82.126:43671)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:12.688 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:12.688 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.82.124:38119(127.9.82.124:38119)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:12.688 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.126:43671: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:12.688 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.124:38119: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:12.690 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345; will retry
07:01:12.707 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.126:43671(127.9.82.126:43671)] connecting to peer
07:01:12.708 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.124:38119(127.9.82.124:38119)] connecting to peer
07:01:12.708 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:12.708 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.82.126:43671(127.9.82.126:43671)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:12.708 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.126:43671: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:12.708 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:12.709 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.82.124:38119(127.9.82.124:38119)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:12.709 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.124:38119: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:12.710 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345; will retry
07:01:12.727 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.126:43671(127.9.82.126:43671)] connecting to peer
07:01:12.728 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.124:38119(127.9.82.124:38119)] connecting to peer
07:01:12.728 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:12.728 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.82.126:43671(127.9.82.126:43671)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:12.728 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:12.728 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.126:43671: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:12.728 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.82.124:38119(127.9.82.124:38119)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:12.728 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.124:38119: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:12.730 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345; will retry
07:01:12.747 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.126:43671(127.9.82.126:43671)] connecting to peer
07:01:12.748 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.124:38119(127.9.82.124:38119)] connecting to peer
07:01:12.748 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:12.748 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.82.126:43671(127.9.82.126:43671)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:12.748 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.126:43671: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:12.749 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:12.749 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.82.124:38119(127.9.82.124:38119)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:12.749 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.124:38119: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:12.750 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345; will retry
07:01:12.767 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.126:43671(127.9.82.126:43671)] connecting to peer
07:01:12.767 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.124:38119(127.9.82.124:38119)] connecting to peer
07:01:12.768 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:12.768 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.82.126:43671(127.9.82.126:43671)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:12.768 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.126:43671: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:12.768 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:12.768 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.82.124:38119(127.9.82.124:38119)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:12.768 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.124:38119: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:12.770 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345; will retry
07:01:12.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:12.818758 10404 raft_consensus.cc:482] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 4a229e6d106c46caad88d051aa87c533)
07:01:12.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:12.818962 10404 raft_consensus.cc:504] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9089901da6614cedb5be9c0f69cef850" member_type: VOTER last_known_addr { host: "127.9.82.126" port: 43671 } } peers { permanent_uuid: "7b5c12662b074d3e903fbcd05ed4a533" member_type: VOTER last_known_addr { host: "127.9.82.125" port: 45345 } } peers { permanent_uuid: "4a229e6d106c46caad88d051aa87c533" member_type: VOTER last_known_addr { host: "127.9.82.124" port: 38119 } }
07:01:12.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:12.820010 10404 leader_election.cc:288] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 9089901da6614cedb5be9c0f69cef850 (127.9.82.126:43671), 4a229e6d106c46caad88d051aa87c533 (127.9.82.124:38119)
07:01:12.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:12.820822 9655 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.82.126:43671: connect: Connection refused (error 111) [suppressed 6 similar messages]
07:01:12.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:12.822054 9654 leader_election.cc:334] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 4a229e6d106c46caad88d051aa87c533 (127.9.82.124:38119): Network error: Client connection negotiation failed: client connection to 127.9.82.124:38119: connect: Connection refused (error 111)
07:01:12.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:12.822221 9655 leader_election.cc:334] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 9089901da6614cedb5be9c0f69cef850 (127.9.82.126:43671): Network error: Client connection negotiation failed: client connection to 127.9.82.126:43671: connect: Connection refused (error 111)
07:01:12.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:12.822351 9655 leader_election.cc:302] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 7b5c12662b074d3e903fbcd05ed4a533; no voters: 4a229e6d106c46caad88d051aa87c533, 9089901da6614cedb5be9c0f69cef850
07:01:12.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:12.822594 10404 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
07:01:12.848 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.126:43671(127.9.82.126:43671)] connecting to peer
07:01:12.848 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.124:38119(127.9.82.124:38119)] connecting to peer
07:01:12.848 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:12.848 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.82.126:43671(127.9.82.126:43671)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:12.849 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.126:43671: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:12.849 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:12.849 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.82.124:38119(127.9.82.124:38119)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:12.849 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.124:38119: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:12.850 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345; will retry
07:01:12.867 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.126:43671(127.9.82.126:43671)] connecting to peer
07:01:12.867 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.124:38119(127.9.82.124:38119)] connecting to peer
07:01:12.868 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:12.868 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.82.126:43671(127.9.82.126:43671)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:12.868 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:12.868 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.126:43671: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:12.868 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.82.124:38119(127.9.82.124:38119)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:12.868 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.124:38119: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:12.870 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345; will retry
07:01:12.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:12.950302 10137 heartbeater.cc:503] Master 127.9.82.124:38119 was elected leader, sending a full tablet report...
07:01:12.968 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.126:43671(127.9.82.126:43671)] connecting to peer
07:01:12.968 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.124:38119(127.9.82.124:38119)] connecting to peer
07:01:12.969 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:12.969 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.82.126:43671(127.9.82.126:43671)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:12.969 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.126:43671: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:12.969 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:12.969 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.82.124:38119(127.9.82.124:38119)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:12.969 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.124:38119: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:12.971 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345; will retry
07:01:13.067 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.126:43671(127.9.82.126:43671)] connecting to peer
07:01:13.067 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.124:38119(127.9.82.124:38119)] connecting to peer
07:01:13.068 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:13.068 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.82.126:43671(127.9.82.126:43671)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:13.068 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.126:43671: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:13.068 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:13.068 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.82.124:38119(127.9.82.124:38119)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:13.068 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.124:38119: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:13.070 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345; will retry
07:01:13.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:13.191394 10404 raft_consensus.cc:482] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 4a229e6d106c46caad88d051aa87c533)
07:01:13.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:13.191641 10404 raft_consensus.cc:504] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9089901da6614cedb5be9c0f69cef850" member_type: VOTER last_known_addr { host: "127.9.82.126" port: 43671 } } peers { permanent_uuid: "7b5c12662b074d3e903fbcd05ed4a533" member_type: VOTER last_known_addr { host: "127.9.82.125" port: 45345 } } peers { permanent_uuid: "4a229e6d106c46caad88d051aa87c533" member_type: VOTER last_known_addr { host: "127.9.82.124" port: 38119 } }
07:01:13.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:13.192371 10404 leader_election.cc:288] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 9089901da6614cedb5be9c0f69cef850 (127.9.82.126:43671), 4a229e6d106c46caad88d051aa87c533 (127.9.82.124:38119)
07:01:13.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:13.193598 9655 leader_election.cc:334] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 9089901da6614cedb5be9c0f69cef850 (127.9.82.126:43671): Network error: Client connection negotiation failed: client connection to 127.9.82.126:43671: connect: Connection refused (error 111)
07:01:13.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:13.193749 9654 leader_election.cc:334] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 4a229e6d106c46caad88d051aa87c533 (127.9.82.124:38119): Network error: Client connection negotiation failed: client connection to 127.9.82.124:38119: connect: Connection refused (error 111)
07:01:13.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:13.193854 9654 leader_election.cc:302] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 7b5c12662b074d3e903fbcd05ed4a533; no voters: 4a229e6d106c46caad88d051aa87c533, 9089901da6614cedb5be9c0f69cef850
07:01:13.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:13.194084 10404 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
07:01:13.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:13.574723 10404 raft_consensus.cc:482] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 4a229e6d106c46caad88d051aa87c533)
07:01:13.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:13.574967 10404 raft_consensus.cc:504] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9089901da6614cedb5be9c0f69cef850" member_type: VOTER last_known_addr { host: "127.9.82.126" port: 43671 } } peers { permanent_uuid: "7b5c12662b074d3e903fbcd05ed4a533" member_type: VOTER last_known_addr { host: "127.9.82.125" port: 45345 } } peers { permanent_uuid: "4a229e6d106c46caad88d051aa87c533" member_type: VOTER last_known_addr { host: "127.9.82.124" port: 38119 } }
07:01:13.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:13.575860 10404 leader_election.cc:288] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 9089901da6614cedb5be9c0f69cef850 (127.9.82.126:43671), 4a229e6d106c46caad88d051aa87c533 (127.9.82.124:38119)
07:01:13.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:13.577129 9654 leader_election.cc:334] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 4a229e6d106c46caad88d051aa87c533 (127.9.82.124:38119): Network error: Client connection negotiation failed: client connection to 127.9.82.124:38119: connect: Connection refused (error 111)
07:01:13.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:13.577270 9655 leader_election.cc:334] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 9089901da6614cedb5be9c0f69cef850 (127.9.82.126:43671): Network error: Client connection negotiation failed: client connection to 127.9.82.126:43671: connect: Connection refused (error 111)
07:01:13.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:13.577370 9655 leader_election.cc:302] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 7b5c12662b074d3e903fbcd05ed4a533; no voters: 4a229e6d106c46caad88d051aa87c533, 9089901da6614cedb5be9c0f69cef850
07:01:13.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:13.577600 10404 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
07:01:13.907 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.126:43671(127.9.82.126:43671)] connecting to peer
07:01:13.908 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.124:38119(127.9.82.124:38119)] connecting to peer
07:01:13.908 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:13.908 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.82.124:38119(127.9.82.124:38119)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:13.908 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:13.908 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.124:38119: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:13.908 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.82.126:43671(127.9.82.126:43671)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:13.909 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.126:43671: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:13.910 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345; will retry
07:01:14.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:14.564880 10416 raft_consensus.cc:482] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 4a229e6d106c46caad88d051aa87c533)
07:01:14.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:14.565093 10416 raft_consensus.cc:504] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9089901da6614cedb5be9c0f69cef850" member_type: VOTER last_known_addr { host: "127.9.82.126" port: 43671 } } peers { permanent_uuid: "7b5c12662b074d3e903fbcd05ed4a533" member_type: VOTER last_known_addr { host: "127.9.82.125" port: 45345 } } peers { permanent_uuid: "4a229e6d106c46caad88d051aa87c533" member_type: VOTER last_known_addr { host: "127.9.82.124" port: 38119 } }
07:01:14.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:14.565891 10416 leader_election.cc:288] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 9089901da6614cedb5be9c0f69cef850 (127.9.82.126:43671), 4a229e6d106c46caad88d051aa87c533 (127.9.82.124:38119)
07:01:14.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:14.567937 9655 leader_election.cc:334] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 9089901da6614cedb5be9c0f69cef850 (127.9.82.126:43671): Network error: Client connection negotiation failed: client connection to 127.9.82.126:43671: connect: Connection refused (error 111)
07:01:14.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:14.568111 9654 leader_election.cc:334] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 4a229e6d106c46caad88d051aa87c533 (127.9.82.124:38119): Network error: Client connection negotiation failed: client connection to 127.9.82.124:38119: connect: Connection refused (error 111)
07:01:14.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:14.568217 9654 leader_election.cc:302] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 7b5c12662b074d3e903fbcd05ed4a533; no voters: 4a229e6d106c46caad88d051aa87c533, 9089901da6614cedb5be9c0f69cef850
07:01:14.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:14.568447 10416 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
07:01:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:15.289594 10422 raft_consensus.cc:482] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 4a229e6d106c46caad88d051aa87c533)
07:01:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:15.289777 10422 raft_consensus.cc:504] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9089901da6614cedb5be9c0f69cef850" member_type: VOTER last_known_addr { host: "127.9.82.126" port: 43671 } } peers { permanent_uuid: "7b5c12662b074d3e903fbcd05ed4a533" member_type: VOTER last_known_addr { host: "127.9.82.125" port: 45345 } } peers { permanent_uuid: "4a229e6d106c46caad88d051aa87c533" member_type: VOTER last_known_addr { host: "127.9.82.124" port: 38119 } }
07:01:15.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:15.290403 10422 leader_election.cc:288] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 9089901da6614cedb5be9c0f69cef850 (127.9.82.126:43671), 4a229e6d106c46caad88d051aa87c533 (127.9.82.124:38119)
07:01:15.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:15.292361 9655 leader_election.cc:334] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 9089901da6614cedb5be9c0f69cef850 (127.9.82.126:43671): Network error: Client connection negotiation failed: client connection to 127.9.82.126:43671: connect: Connection refused (error 111)
07:01:15.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:15.292537 9654 leader_election.cc:334] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 4a229e6d106c46caad88d051aa87c533 (127.9.82.124:38119): Network error: Client connection negotiation failed: client connection to 127.9.82.124:38119: connect: Connection refused (error 111)
07:01:15.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:15.292649 9654 leader_election.cc:302] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 7b5c12662b074d3e903fbcd05ed4a533; no voters: 4a229e6d106c46caad88d051aa87c533, 9089901da6614cedb5be9c0f69cef850
07:01:15.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:15.292876 10422 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
07:01:15.928 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.126:43671(127.9.82.126:43671)] connecting to peer
07:01:15.928 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.124:38119(127.9.82.124:38119)] connecting to peer
07:01:15.929 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:15.929 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.82.124:38119(127.9.82.124:38119)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:15.929 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:15.929 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.124:38119: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:15.929 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.82.126:43671(127.9.82.126:43671)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:15.929 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.126:43671: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:15.930 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345; will retry
07:01:16.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:16.905725 10433 raft_consensus.cc:482] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 4a229e6d106c46caad88d051aa87c533)
07:01:16.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:16.905936 10433 raft_consensus.cc:504] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9089901da6614cedb5be9c0f69cef850" member_type: VOTER last_known_addr { host: "127.9.82.126" port: 43671 } } peers { permanent_uuid: "7b5c12662b074d3e903fbcd05ed4a533" member_type: VOTER last_known_addr { host: "127.9.82.125" port: 45345 } } peers { permanent_uuid: "4a229e6d106c46caad88d051aa87c533" member_type: VOTER last_known_addr { host: "127.9.82.124" port: 38119 } }
07:01:16.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:16.906610 10433 leader_election.cc:288] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 9089901da6614cedb5be9c0f69cef850 (127.9.82.126:43671), 4a229e6d106c46caad88d051aa87c533 (127.9.82.124:38119)
07:01:16.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:16.908704 9654 leader_election.cc:334] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 4a229e6d106c46caad88d051aa87c533 (127.9.82.124:38119): Network error: Client connection negotiation failed: client connection to 127.9.82.124:38119: connect: Connection refused (error 111)
07:01:16.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:16.908864 9655 leader_election.cc:334] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 9089901da6614cedb5be9c0f69cef850 (127.9.82.126:43671): Network error: Client connection negotiation failed: client connection to 127.9.82.126:43671: connect: Connection refused (error 111)
07:01:16.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:16.908964 9655 leader_election.cc:302] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 7b5c12662b074d3e903fbcd05ed4a533; no voters: 4a229e6d106c46caad88d051aa87c533, 9089901da6614cedb5be9c0f69cef850
07:01:16.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:16.909214 10433 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
07:01:17.608 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.126:43671(127.9.82.126:43671)] connecting to peer
07:01:17.608 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.124:38119(127.9.82.124:38119)] connecting to peer
07:01:17.609 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:17.609 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.82.126:43671(127.9.82.126:43671)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:17.609 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.126:43671: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:17.609 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:17.609 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.82.124:38119(127.9.82.124:38119)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:17.609 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.124:38119: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:17.611 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345; will retry
07:01:17.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:17.812417 10443 raft_consensus.cc:482] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 4a229e6d106c46caad88d051aa87c533)
07:01:17.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:17.812633 10443 raft_consensus.cc:504] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9089901da6614cedb5be9c0f69cef850" member_type: VOTER last_known_addr { host: "127.9.82.126" port: 43671 } } peers { permanent_uuid: "7b5c12662b074d3e903fbcd05ed4a533" member_type: VOTER last_known_addr { host: "127.9.82.125" port: 45345 } } peers { permanent_uuid: "4a229e6d106c46caad88d051aa87c533" member_type: VOTER last_known_addr { host: "127.9.82.124" port: 38119 } }
07:01:17.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:17.813318 10443 leader_election.cc:288] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 9089901da6614cedb5be9c0f69cef850 (127.9.82.126:43671), 4a229e6d106c46caad88d051aa87c533 (127.9.82.124:38119)
07:01:17.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:17.815337 9654 leader_election.cc:334] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 4a229e6d106c46caad88d051aa87c533 (127.9.82.124:38119): Network error: Client connection negotiation failed: client connection to 127.9.82.124:38119: connect: Connection refused (error 111)
07:01:17.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:17.815519 9655 leader_election.cc:334] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 9089901da6614cedb5be9c0f69cef850 (127.9.82.126:43671): Network error: Client connection negotiation failed: client connection to 127.9.82.126:43671: connect: Connection refused (error 111)
07:01:17.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:17.815611 9655 leader_election.cc:302] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 7b5c12662b074d3e903fbcd05ed4a533; no voters: 4a229e6d106c46caad88d051aa87c533, 9089901da6614cedb5be9c0f69cef850
07:01:17.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:17.815913 10443 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
07:01:19.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:19.961220 10456 raft_consensus.cc:482] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 4a229e6d106c46caad88d051aa87c533)
07:01:19.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:19.961441 10456 raft_consensus.cc:504] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9089901da6614cedb5be9c0f69cef850" member_type: VOTER last_known_addr { host: "127.9.82.126" port: 43671 } } peers { permanent_uuid: "7b5c12662b074d3e903fbcd05ed4a533" member_type: VOTER last_known_addr { host: "127.9.82.125" port: 45345 } } peers { permanent_uuid: "4a229e6d106c46caad88d051aa87c533" member_type: VOTER last_known_addr { host: "127.9.82.124" port: 38119 } }
07:01:19.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:19.962229 10456 leader_election.cc:288] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 9089901da6614cedb5be9c0f69cef850 (127.9.82.126:43671), 4a229e6d106c46caad88d051aa87c533 (127.9.82.124:38119)
07:01:19.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:19.963178 9655 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.82.126:43671: connect: Connection refused (error 111) [suppressed 13 similar messages]
07:01:19.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:19.965188 9654 leader_election.cc:334] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 4a229e6d106c46caad88d051aa87c533 (127.9.82.124:38119): Network error: Client connection negotiation failed: client connection to 127.9.82.124:38119: connect: Connection refused (error 111)
07:01:19.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:19.965888 9655 leader_election.cc:334] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 9089901da6614cedb5be9c0f69cef850 (127.9.82.126:43671): Network error: Client connection negotiation failed: client connection to 127.9.82.126:43671: connect: Connection refused (error 111)
07:01:19.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:19.966070 9655 leader_election.cc:302] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 7b5c12662b074d3e903fbcd05ed4a533; no voters: 4a229e6d106c46caad88d051aa87c533, 9089901da6614cedb5be9c0f69cef850
07:01:19.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:19.966404 10456 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
07:01:21.648 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.126:43671(127.9.82.126:43671)] connecting to peer
07:01:21.648 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.124:38119(127.9.82.124:38119)] connecting to peer
07:01:21.649 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:21.649 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.82.126:43671(127.9.82.126:43671)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:21.649 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.126:43671: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:21.649 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:21.649 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.82.124:38119(127.9.82.124:38119)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:21.649 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.124:38119: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:21.651 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345; will retry
07:01:23.988 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.126:43671(127.9.82.126:43671)] connecting to peer
07:01:23.988 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.124:38119(127.9.82.124:38119)] connecting to peer
07:01:23.989 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:23.989 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:23.989 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.82.124:38119(127.9.82.124:38119)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:23.989 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.82.126:43671(127.9.82.126:43671)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:23.989 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.126:43671: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:23.989 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.124:38119: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:23.990 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345; will retry
07:01:24.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:24.669832 10484 raft_consensus.cc:482] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 4a229e6d106c46caad88d051aa87c533)
07:01:24.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:24.670069 10484 raft_consensus.cc:504] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9089901da6614cedb5be9c0f69cef850" member_type: VOTER last_known_addr { host: "127.9.82.126" port: 43671 } } peers { permanent_uuid: "7b5c12662b074d3e903fbcd05ed4a533" member_type: VOTER last_known_addr { host: "127.9.82.125" port: 45345 } } peers { permanent_uuid: "4a229e6d106c46caad88d051aa87c533" member_type: VOTER last_known_addr { host: "127.9.82.124" port: 38119 } }
07:01:24.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:24.670794 10484 leader_election.cc:288] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 9089901da6614cedb5be9c0f69cef850 (127.9.82.126:43671), 4a229e6d106c46caad88d051aa87c533 (127.9.82.124:38119)
07:01:24.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:24.672909 9655 leader_election.cc:334] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 9089901da6614cedb5be9c0f69cef850 (127.9.82.126:43671): Network error: Client connection negotiation failed: client connection to 127.9.82.126:43671: connect: Connection refused (error 111)
07:01:24.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:24.673072 9654 leader_election.cc:334] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 4a229e6d106c46caad88d051aa87c533 (127.9.82.124:38119): Network error: Client connection negotiation failed: client connection to 127.9.82.124:38119: connect: Connection refused (error 111)
07:01:24.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:24.673182 9654 leader_election.cc:302] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 7b5c12662b074d3e903fbcd05ed4a533; no voters: 4a229e6d106c46caad88d051aa87c533, 9089901da6614cedb5be9c0f69cef850
07:01:24.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:24.673398 10484 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
07:01:27.987 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.126:43671(127.9.82.126:43671)] connecting to peer
07:01:27.988 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.124:38119(127.9.82.124:38119)] connecting to peer
07:01:27.989 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:27.989 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.82.126:43671(127.9.82.126:43671)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:27.989 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:27.989 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.126:43671: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:27.989 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.82.124:38119(127.9.82.124:38119)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:27.989 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.124:38119: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:27.990 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345; will retry
07:01:30.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:30.959686 10517 raft_consensus.cc:482] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 4a229e6d106c46caad88d051aa87c533)
07:01:30.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:30.959882 10517 raft_consensus.cc:504] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9089901da6614cedb5be9c0f69cef850" member_type: VOTER last_known_addr { host: "127.9.82.126" port: 43671 } } peers { permanent_uuid: "7b5c12662b074d3e903fbcd05ed4a533" member_type: VOTER last_known_addr { host: "127.9.82.125" port: 45345 } } peers { permanent_uuid: "4a229e6d106c46caad88d051aa87c533" member_type: VOTER last_known_addr { host: "127.9.82.124" port: 38119 } }
07:01:30.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:30.960482 10517 leader_election.cc:288] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 9089901da6614cedb5be9c0f69cef850 (127.9.82.126:43671), 4a229e6d106c46caad88d051aa87c533 (127.9.82.124:38119)
07:01:30.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:30.961416 9655 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.82.126:43671: connect: Connection refused (error 111) [suppressed 3 similar messages]
07:01:30.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:30.962428 9654 leader_election.cc:334] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 4a229e6d106c46caad88d051aa87c533 (127.9.82.124:38119): Network error: Client connection negotiation failed: client connection to 127.9.82.124:38119: connect: Connection refused (error 111)
07:01:30.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:30.962661 9655 leader_election.cc:334] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 9089901da6614cedb5be9c0f69cef850 (127.9.82.126:43671): Network error: Client connection negotiation failed: client connection to 127.9.82.126:43671: connect: Connection refused (error 111)
07:01:30.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:30.962808 9655 leader_election.cc:302] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 7b5c12662b074d3e903fbcd05ed4a533; no voters: 4a229e6d106c46caad88d051aa87c533, 9089901da6614cedb5be9c0f69cef850
07:01:30.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:30.963115 10517 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
07:01:31.787 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.126:43671(127.9.82.126:43671)] connecting to peer
07:01:31.788 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.124:38119(127.9.82.124:38119)] connecting to peer
07:01:31.788 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:31.788 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.82.126:43671(127.9.82.126:43671)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:31.788 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:31.788 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.82.124:38119(127.9.82.124:38119)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:31.788 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.126:43671: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:31.789 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.124:38119: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:31.791 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345; will retry
07:01:32.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:32.301795 10528 raft_consensus.cc:482] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 4a229e6d106c46caad88d051aa87c533)
07:01:32.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:32.302035 10528 raft_consensus.cc:504] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9089901da6614cedb5be9c0f69cef850" member_type: VOTER last_known_addr { host: "127.9.82.126" port: 43671 } } peers { permanent_uuid: "7b5c12662b074d3e903fbcd05ed4a533" member_type: VOTER last_known_addr { host: "127.9.82.125" port: 45345 } } peers { permanent_uuid: "4a229e6d106c46caad88d051aa87c533" member_type: VOTER last_known_addr { host: "127.9.82.124" port: 38119 } }
07:01:32.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:32.302969 10528 leader_election.cc:288] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 9089901da6614cedb5be9c0f69cef850 (127.9.82.126:43671), 4a229e6d106c46caad88d051aa87c533 (127.9.82.124:38119)
07:01:32.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:32.304936 9655 leader_election.cc:334] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 9089901da6614cedb5be9c0f69cef850 (127.9.82.126:43671): Network error: Client connection negotiation failed: client connection to 127.9.82.126:43671: connect: Connection refused (error 111)
07:01:32.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:32.305131 9654 leader_election.cc:334] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 4a229e6d106c46caad88d051aa87c533 (127.9.82.124:38119): Network error: Client connection negotiation failed: client connection to 127.9.82.124:38119: connect: Connection refused (error 111)
07:01:32.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:32.305208 9654 leader_election.cc:302] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 7b5c12662b074d3e903fbcd05ed4a533; no voters: 4a229e6d106c46caad88d051aa87c533, 9089901da6614cedb5be9c0f69cef850
07:01:32.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:32.305450 10528 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
07:01:33.747 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.126:43671(127.9.82.126:43671)] connecting to peer
07:01:33.748 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.124:38119(127.9.82.124:38119)] connecting to peer
07:01:33.748 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:33.748 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.82.126:43671(127.9.82.126:43671)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:33.748 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.126:43671: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:33.748 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:33.748 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.82.124:38119(127.9.82.124:38119)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:33.749 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.124:38119: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:33.750 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345; will retry
07:01:37.028 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.126:43671(127.9.82.126:43671)] connecting to peer
07:01:37.028 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.124:38119(127.9.82.124:38119)] connecting to peer
07:01:37.029 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:37.029 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:37.029 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.82.126:43671(127.9.82.126:43671)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:37.029 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.82.124:38119(127.9.82.124:38119)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:37.029 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.124:38119: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:37.029 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.126:43671: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:37.031 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345; will retry
07:01:38.607 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.126:43671(127.9.82.126:43671)] connecting to peer
07:01:38.608 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.124:38119(127.9.82.124:38119)] connecting to peer
07:01:38.608 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:38.608 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.82.126:43671(127.9.82.126:43671)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:38.608 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.126:43671: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:38.609 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:38.609 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.82.124:38119(127.9.82.124:38119)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:38.609 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.124:38119: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:38.610 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345; will retry
07:01:38.768 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.126:43671(127.9.82.126:43671)] connecting to peer
07:01:38.768 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.124:38119(127.9.82.124:38119)] connecting to peer
07:01:38.769 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:38.769 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.82.126:43671(127.9.82.126:43671)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:38.769 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.126:43671: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:38.769 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:38.769 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.82.124:38119(127.9.82.124:38119)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:38.769 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.124:38119: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:38.770 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345; will retry
07:01:42.807 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.126:43671(127.9.82.126:43671)] connecting to peer
07:01:42.807 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.124:38119(127.9.82.124:38119)] connecting to peer
07:01:42.808 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:42.808 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.82.126:43671(127.9.82.126:43671)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:42.808 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.126:43671: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:42.808 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:42.808 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.82.124:38119(127.9.82.124:38119)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:42.808 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.124:38119: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:42.810 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345; will retry
07:01:46.047 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.126:43671(127.9.82.126:43671)] connecting to peer
07:01:46.047 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.124:38119(127.9.82.124:38119)] connecting to peer
07:01:46.048 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:46.048 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.82.126:43671(127.9.82.126:43671)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:46.048 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:46.048 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.126:43671: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:46.048 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.82.124:38119(127.9.82.124:38119)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:46.048 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.124:38119: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:46.050 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345; will retry
07:01:47.987 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.126:43671(127.9.82.126:43671)] connecting to peer
07:01:47.988 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.124:38119(127.9.82.124:38119)] connecting to peer
07:01:47.988 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:47.988 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.82.124:38119(127.9.82.124:38119)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:47.988 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:47.988 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.124:38119: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:47.988 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.82.126:43671(127.9.82.126:43671)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:47.989 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.126:43671: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:47.990 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345; will retry
07:01:49.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:49.132933 10627 raft_consensus.cc:482] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 4a229e6d106c46caad88d051aa87c533)
07:01:49.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:49.133157 10627 raft_consensus.cc:504] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9089901da6614cedb5be9c0f69cef850" member_type: VOTER last_known_addr { host: "127.9.82.126" port: 43671 } } peers { permanent_uuid: "7b5c12662b074d3e903fbcd05ed4a533" member_type: VOTER last_known_addr { host: "127.9.82.125" port: 45345 } } peers { permanent_uuid: "4a229e6d106c46caad88d051aa87c533" member_type: VOTER last_known_addr { host: "127.9.82.124" port: 38119 } }
07:01:49.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:49.134198 10627 leader_election.cc:288] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 9089901da6614cedb5be9c0f69cef850 (127.9.82.126:43671), 4a229e6d106c46caad88d051aa87c533 (127.9.82.124:38119)
07:01:49.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:49.135545 9654 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.82.124:38119: connect: Connection refused (error 111) [suppressed 3 similar messages]
07:01:49.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:49.136976 9655 leader_election.cc:334] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 9089901da6614cedb5be9c0f69cef850 (127.9.82.126:43671): Network error: Client connection negotiation failed: client connection to 127.9.82.126:43671: connect: Connection refused (error 111)
07:01:49.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:49.137145 9654 leader_election.cc:334] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 4a229e6d106c46caad88d051aa87c533 (127.9.82.124:38119): Network error: Client connection negotiation failed: client connection to 127.9.82.124:38119: connect: Connection refused (error 111)
07:01:49.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:49.137328 9654 leader_election.cc:302] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 7b5c12662b074d3e903fbcd05ed4a533; no voters: 4a229e6d106c46caad88d051aa87c533, 9089901da6614cedb5be9c0f69cef850
07:01:49.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:49.137674 10627 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 7b5c12662b074d3e903fbcd05ed4a533 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
07:01:51.627 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.126:43671(127.9.82.126:43671)] connecting to peer
07:01:51.627 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.124:38119(127.9.82.124:38119)] connecting to peer
07:01:51.628 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:51.628 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:51.628 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.82.124:38119(127.9.82.124:38119)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:51.628 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.82.126:43671(127.9.82.126:43671)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:51.628 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.124:38119: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:51.628 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.126:43671: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:51.629 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345; will retry
07:01:51.967 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.126:43671(127.9.82.126:43671)] connecting to peer
07:01:51.968 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.124:38119(127.9.82.124:38119)] connecting to peer
07:01:51.969 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:51.969 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:51.969 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.82.126:43671(127.9.82.126:43671)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:51.969 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.82.124:38119(127.9.82.124:38119)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:51.969 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.126:43671: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:51.969 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.124:38119: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:51.970 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345; will retry
07:01:54.467 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.126:43671(127.9.82.126:43671)] connecting to peer
07:01:54.467 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.124:38119(127.9.82.124:38119)] connecting to peer
07:01:54.468 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:54.468 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.82.126:43671(127.9.82.126:43671)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:54.468 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:54.468 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.82.124:38119(127.9.82.124:38119)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:54.468 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.126:43671: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:54.468 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.124:38119: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:54.470 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345; will retry
07:01:56.167 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.126:43671(127.9.82.126:43671)] connecting to peer
07:01:56.168 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.124:38119(127.9.82.124:38119)] connecting to peer
07:01:56.168 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:56.168 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.82.126:43671(127.9.82.126:43671)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:56.168 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:56.168 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.126:43671: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:56.168 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.82.124:38119(127.9.82.124:38119)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:56.169 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.124:38119: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:56.170 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345; will retry
07:01:59.287 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.126:43671(127.9.82.126:43671)] connecting to peer
07:01:59.288 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.9.82.124:38119(127.9.82.124:38119)] connecting to peer
07:01:59.288 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:59.288 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.82.126:43671(127.9.82.126:43671)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:59.288 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:59.288 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.126:43671: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671
07:01:59.288 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.82.124:38119(127.9.82.124:38119)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:59.288 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.82.124:38119: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
07:01:59.290 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345; will retry
07:01:59.291 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=CommitTransaction, tablet=null, attempt=29, TimeoutTracker(timeout=50000, elapsed=46615), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [1ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [1ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: Network error: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [3ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [12ms] refreshing cache from master, [12ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [12ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [13ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [13ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [13ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: Network error: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [14ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [15ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [32ms] refreshing cache from master, [32ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [33ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [33ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [33ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [34ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: Network error: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [35ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [35ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [52ms] refreshing cache from master, [52ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [53ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [53ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [53ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [53ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: Network error: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [55ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [55ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [72ms] refreshing cache from master, [72ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [73ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [73ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [74ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [74ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: Network error: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [75ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [75ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [92ms] refreshing cache from master, [92ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [92ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [93ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [93ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [93ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: Network error: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [95ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [95ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [172ms] refreshing cache from master, [172ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [173ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [173ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [174ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [174ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: Network error: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [175ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [175ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [191ms] refreshing cache from master, [192ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [192ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [193ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [193ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [193ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: Network error: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [194ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [195ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [292ms] refreshing cache from master, [293ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [293ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [294ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [294ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [294ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: Network error: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [296ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [296ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [391ms] refreshing cache from master, [392ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [392ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [393ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [393ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [393ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: Network error: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [395ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [395ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [1232ms] refreshing cache from master, [1232ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [1233ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [1233ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [1233ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: Network error: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [1234ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [1235ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [1235ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [3252ms] refreshing cache from master, [3252ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [3253ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [3253ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [3254ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: Network error: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [3254ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [3255ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [3255ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [4932ms] refreshing cache from master, [4932ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [4933ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [4933ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [4934ms] too many traces: truncated at 100 traces, deferred=Deferred@734488831(state=PENDING, result=null, callback=(continuation of Deferred@608359062 after retry RPC after error@1090985375) -> (continuation of Deferred@146524969 after retry RPC after error@405527121) -> (continuation of Deferred@718237088 after retry RPC after error@1398630019) -> (continuation of Deferred@90770754 after retry RPC after error@1138145004) -> (continuation of Deferred@304439643 after retry RPC after error@720659682) -> (continuation of Deferred@1731887075 after retry RPC after error@288216126) -> (continuation of Deferred@683364601 after retry RPC after error@568455800) -> (continuation of Deferred@353387712 after retry RPC after error@1784938104) -> (continuation of Deferred@414013269 after retry RPC after error@913376266) -> (continuation of Deferred@558919744 after retry RPC after error@1421204012) -> (continuation of Deferred@642791440 after retry RPC after error@415181188) -> (continuation of Deferred@986679459 after retry RPC after error@1079066988) -> (continuation of Deferred@1743151566 after retry RPC after error@116692927) -> (continuation of Deferred@1085840722 after retry RPC after error@170454536) -> (continuation of Deferred@1586759314 after retry RPC after error@2097101712) -> (continuation of Deferred@2015984464 after retry RPC after error@901546739) -> (continuation of Deferred@1264755264 after retry RPC after error@1313088263) -> (continuation of Deferred@2105815652 after retry RPC after error@746495037) -> (continuation of Deferred@1926257488 after retry RPC after error@493031958) -> (continuation of Deferred@1101530316 after retry RPC after error@426764208) -> (continuation of Deferred@1771386134 after retry RPC after error@1826847120) -> (continuation of Deferred@1835835044 after retry RPC after error@190359262) -> (continuation of Deferred@1260724006 after retry RPC after error@1014874155) -> (continuation of Deferred@904782014 after retry RPC after error@2068700811) -> (continuation of Deferred@418525931 after retry RPC after error@1247593798) -> (continuation of Deferred@284046499 after retry RPC after error@96602547) -> (continuation of Deferred@212735626 after retry RPC after error@544507052) -> (continuation of Deferred@1096738605 after retry RPC after error@1413804324), errback=(continuation of Deferred@608359062 after retry RPC after error@1090985375) -> (continuation of Deferred@146524969 after retry RPC after error@405527121) -> (continuation of Deferred@718237088 after retry RPC after error@1398630019) -> (continuation of Deferred@90770754 after retry RPC after error@1138145004) -> (continuation of Deferred@304439643 after retry RPC after error@720659682) -> (continuation of Deferred@1731887075 after retry RPC after error@288216126) -> (continuation of Deferred@683364601 after retry RPC after error@568455800) -> (continuation of Deferred@353387712 after retry RPC after error@1784938104) -> (continuation of Deferred@414013269 after retry RPC after error@913376266) -> (continuation of Deferred@558919744 after retry RPC after error@1421204012) -> (continuation of Deferred@642791440 after retry RPC after error@415181188) -> (continuation of Deferred@986679459 after retry RPC after error@1079066988) -> (continuation of Deferred@1743151566 after retry RPC after error@116692927) -> (continuation of Deferred@1085840722 after retry RPC after error@170454536) -> (continuation of Deferred@1586759314 after retry RPC after error@2097101712) -> (continuation of Deferred@2015984464 after retry RPC after error@901546739) -> (continuation of Deferred@1264755264 after retry RPC after error@1313088263) -> (continuation of Deferred@2105815652 after retry RPC after error@746495037) -> (continuation of Deferred@1926257488 after retry RPC after error@493031958) -> (continuation of Deferred@1101530316 after retry RPC after error@426764208) -> (continuation of Deferred@1771386134 after retry RPC after error@1826847120) -> (continuation of Deferred@1835835044 after retry RPC after error@190359262) -> (continuation of Deferred@1260724006 after retry RPC after error@1014874155) -> (continuation of Deferred@904782014 after retry RPC after error@2068700811) -> (continuation of Deferred@418525931 after retry RPC after error@1247593798) -> (continuation of Deferred@284046499 after retry RPC after error@96602547) -> (continuation of Deferred@212735626 after retry RPC after error@544507052) -> (continuation of Deferred@1096738605 after retry RPC after error@1413804324)))
07:01:59.294 [DEBUG - main] (Connection.java:575) [peer master-127.9.82.125:45345(127.9.82.125:45345)] disconnecting while in state READY
07:01:59.294 [DEBUG - main] (Connection.java:575) [peer master-127.9.82.124:38119(127.9.82.124:38119)] disconnecting while in state TERMINATED
07:01:59.294 [DEBUG - main] (Connection.java:575) [peer 5289e655d9c647f499e5df5f236ce008(127.9.82.67:36991)] disconnecting while in state READY
07:01:59.294 [DEBUG - main] (Connection.java:575) [peer master-127.9.82.126:43671(127.9.82.126:43671)] disconnecting while in state TERMINATED
07:01:59.295 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 5289e655d9c647f499e5df5f236ce008(127.9.82.67:36991)] handling channelInactive
07:01:59.296 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 5289e655d9c647f499e5df5f236ce008(127.9.82.67:36991)] cleaning up while in state READY due to: connection closed
07:01:59.296 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
07:01:59.297 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.9.82.125:45345(127.9.82.125:45345)] handling channelInactive
07:01:59.297 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.82.125:45345(127.9.82.125:45345)] cleaning up while in state READY due to: connection closed
07:01:59.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:59.298626 9545 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu pid 9836
07:01:59.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:59.326305 9990 generic_service.cc:196] Checking for leaks (request via RPC)
07:01:59.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:59.467953 9545 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu with pid 9836
07:01:59.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:59.490044 9545 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu pid 9998
07:01:59.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:59.496960 10132 generic_service.cc:196] Checking for leaks (request via RPC)
07:01:59.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:59.567468 10158 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.82.65:46241: connect: Connection refused (error 111)
07:01:59.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:01:59.569159 10158 consensus_peers.cc:485] T 33dd825246c541e58f1e54d53dce8b17 P 5289e655d9c647f499e5df5f236ce008 -> Peer 11ae612cdfd94d638c0c35046ab8118d (127.9.82.65:46241): Couldn't send request to peer 11ae612cdfd94d638c0c35046ab8118d. Status: Network error: Client connection negotiation failed: client connection to 127.9.82.65:46241: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
07:01:59.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:59.604491 9545 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu with pid 9998
07:01:59.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:59.623683 9545 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu pid 10141
07:01:59.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:59.630494 10273 generic_service.cc:196] Checking for leaks (request via RPC)
07:01:59.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:59.767906 9545 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu with pid 10141
07:01:59.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:59.791515 9545 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu pid 9634
07:01:59.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:59.792709 9708 generic_service.cc:196] Checking for leaks (request via RPC)
07:01:59.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:01:59.915009 9545 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskjQXd4I/build/asan/bin/kudu with pid 9634
07:01:59.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T07:01:59Z chronyd exiting
07:02:00.073 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveSwitchesToOtherTxnManager: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=CommitTransaction, tablet=null, attempt=29, TimeoutTracker(timeout=50000, elapsed=46615), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [1ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [1ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: Network error: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [3ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [12ms] refreshing cache from master, [12ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [12ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [13ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [13ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [13ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: Network error: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [14ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [15ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [32ms] refreshing cache from master, [32ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [33ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [33ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [33ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [34ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: Network error: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [35ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [35ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [52ms] refreshing cache from master, [52ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [53ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [53ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [53ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [53ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: Network error: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [55ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [55ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [72ms] refreshing cache from master, [72ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [73ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [73ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [74ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [74ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: Network error: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [75ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [75ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [92ms] refreshing cache from master, [92ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [92ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [93ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [93ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [93ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: Network error: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [95ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [95ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [172ms] refreshing cache from master, [172ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [173ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [173ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [174ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [174ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: Network error: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [175ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [175ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [191ms] refreshing cache from master, [192ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [192ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [193ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [193ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [193ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: Network error: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [194ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [195ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [292ms] refreshing cache from master, [293ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [293ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [294ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [294ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [294ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: Network error: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [296ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [296ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [391ms] refreshing cache from master, [392ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [392ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [393ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [393ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [393ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: Network error: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [395ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [395ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [1232ms] refreshing cache from master, [1232ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [1233ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [1233ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [1233ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: Network error: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [1234ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [1235ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [1235ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [3252ms] refreshing cache from master, [3252ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [3253ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [3253ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [3254ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: Network error: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [3254ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [3255ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [3255ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [4932ms] refreshing cache from master, [4932ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [4933ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [4933ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [4934ms] too many traces: truncated at 100 traces, deferred=Deferred@734488831(state=PENDING, result=null, callback=(continuation of Deferred@608359062 after retry RPC after error@1090985375) -> (continuation of Deferred@146524969 after retry RPC after error@405527121) -> (continuation of Deferred@718237088 after retry RPC after error@1398630019) -> (continuation of Deferred@90770754 after retry RPC after error@1138145004) -> (continuation of Deferred@304439643 after retry RPC after error@720659682) -> (continuation of Deferred@1731887075 after retry RPC after error@288216126) -> (continuation of Deferred@683364601 after retry RPC after error@568455800) -> (continuation of Deferred@353387712 after retry RPC after error@1784938104) -> (continuation of Deferred@414013269 after retry RPC after error@913376266) -> (continuation of Deferred@558919744 after retry RPC after error@1421204012) -> (continuation of Deferred@642791440 after retry RPC after error@415181188) -> (continuation of Deferred@986679459 after retry RPC after error@1079066988) -> (continuation of Deferred@1743151566 after retry RPC after error@116692927) -> (continuation of Deferred@1085840722 after retry RPC after error@170454536) -> (continuation of Deferred@1586759314 after retry RPC after error@2097101712) -> (continuation of Deferred@2015984464 after retry RPC after error@901546739) -> (continuation of Deferred@1264755264 after retry RPC after error@1313088263) -> (continuation of Deferred@2105815652 after retry RPC after error@746495037) -> (continuation of Deferred@1926257488 after retry RPC after error@493031958) -> (continuation of Deferred@1101530316 after retry RPC after error@426764208) -> (continuation of Deferred@1771386134 after retry RPC after error@1826847120) -> (continuation of Deferred@1835835044 after retry RPC after error@190359262) -> (continuation of Deferred@1260724006 after retry RPC after error@1014874155) -> (continuation of Deferred@904782014 after retry RPC after error@2068700811) -> (continuation of Deferred@418525931 after retry RPC after error@1247593798) -> (continuation of Deferred@284046499 after retry RPC after error@96602547) -> (continuation of Deferred@212735626 after retry RPC after error@544507052) -> (continuation of Deferred@1096738605 after retry RPC after error@1413804324), errback=(continuation of Deferred@608359062 after retry RPC after error@1090985375) -> (continuation of Deferred@146524969 after retry RPC after error@405527121) -> (continuation of Deferred@718237088 after retry RPC after error@1398630019) -> (continuation of Deferred@90770754 after retry RPC after error@1138145004) -> (continuation of Deferred@304439643 after retry RPC after error@720659682) -> (continuation of Deferred@1731887075 after retry RPC after error@288216126) -> (continuation of Deferred@683364601 after retry RPC after error@568455800) -> (continuation of Deferred@353387712 after retry RPC after error@1784938104) -> (continuation of Deferred@414013269 after retry RPC after error@913376266) -> (continuation of Deferred@558919744 after retry RPC after error@1421204012) -> (continuation of Deferred@642791440 after retry RPC after error@415181188) -> (continuation of Deferred@986679459 after retry RPC after error@1079066988) -> (continuation of Deferred@1743151566 after retry RPC after error@116692927) -> (continuation of Deferred@1085840722 after retry RPC after error@170454536) -> (continuation of Deferred@1586759314 after retry RPC after error@2097101712) -> (continuation of Deferred@2015984464 after retry RPC after error@901546739) -> (continuation of Deferred@1264755264 after retry RPC after error@1313088263) -> (continuation of Deferred@2105815652 after retry RPC after error@746495037) -> (continuation of Deferred@1926257488 after retry RPC after error@493031958) -> (continuation of Deferred@1101530316 after retry RPC after error@426764208) -> (continuation of Deferred@1771386134 after retry RPC after error@1826847120) -> (continuation of Deferred@1835835044 after retry RPC after error@190359262) -> (continuation of Deferred@1260724006 after retry RPC after error@1014874155) -> (continuation of Deferred@904782014 after retry RPC after error@2068700811) -> (continuation of Deferred@418525931 after retry RPC after error@1247593798) -> (continuation of Deferred@284046499 after retry RPC after error@96602547) -> (continuation of Deferred@212735626 after retry RPC after error@544507052) -> (continuation of Deferred@1096738605 after retry RPC after error@1413804324)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduTransaction.doCommitTransaction(KuduTransaction.java:491) ~[main/:?]
at org.apache.kudu.client.KuduTransaction.commitWithMode(KuduTransaction.java:537) ~[main/:?]
at org.apache.kudu.client.KuduTransaction.commit(KuduTransaction.java:284) ~[main/:?]
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveSwitchesToOtherTxnManager(TestKuduTransaction.java:1152) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.9.82.124:38119
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_275]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_275]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=CommitTransaction, tablet=null, attempt=29, TimeoutTracker(timeout=50000, elapsed=46615), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [1ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [1ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: Network error: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [3ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [12ms] refreshing cache from master, [12ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [12ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [13ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [13ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [13ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: Network error: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [14ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [15ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [32ms] refreshing cache from master, [32ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [33ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [33ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [33ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [34ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: Network error: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [35ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [35ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [52ms] refreshing cache from master, [52ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [53ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [53ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [53ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [53ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: Network error: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [55ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [55ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [72ms] refreshing cache from master, [72ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [73ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [73ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [74ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [74ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: Network error: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [75ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [75ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [92ms] refreshing cache from master, [92ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [92ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [93ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [93ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [93ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: Network error: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [95ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [95ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [172ms] refreshing cache from master, [172ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [173ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [173ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [174ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [174ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: Network error: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [175ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [175ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [191ms] refreshing cache from master, [192ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [192ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [193ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [193ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [193ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: Network error: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [194ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [195ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [292ms] refreshing cache from master, [293ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [293ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [294ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [294ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [294ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: Network error: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [296ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [296ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [391ms] refreshing cache from master, [392ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [392ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [393ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [393ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [393ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: Network error: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [395ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [395ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [1232ms] refreshing cache from master, [1232ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [1233ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [1233ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [1233ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: Network error: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [1234ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [1235ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [1235ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [3252ms] refreshing cache from master, [3252ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [3253ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [3253ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [3254ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.124:38119: Network error: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119, [3254ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.126:43671: Network error: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [3255ms] Sub RPC ConnectToMaster: received response from server master-127.9.82.125:45345: OK, [3255ms] delaying RPC due to: Service unavailable: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671, [4932ms] refreshing cache from master, [4932ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.126:43671, [4933ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.124:38119, [4933ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.82.125:45345, [4934ms] too many traces: truncated at 100 traces, deferred=Deferred@734488831(state=PENDING, result=null, callback=(continuation of Deferred@608359062 after retry RPC after error@1090985375) -> (continuation of Deferred@146524969 after retry RPC after error@405527121) -> (continuation of Deferred@718237088 after retry RPC after error@1398630019) -> (continuation of Deferred@90770754 after retry RPC after error@1138145004) -> (continuation of Deferred@304439643 after retry RPC after error@720659682) -> (continuation of Deferred@1731887075 after retry RPC after error@288216126) -> (continuation of Deferred@683364601 after retry RPC after error@568455800) -> (continuation of Deferred@353387712 after retry RPC after error@1784938104) -> (continuation of Deferred@414013269 after retry RPC after error@913376266) -> (continuation of Deferred@558919744 after retry RPC after error@1421204012) -> (continuation of Deferred@642791440 after retry RPC after error@415181188) -> (continuation of Deferred@986679459 after retry RPC after error@1079066988) -> (continuation of Deferred@1743151566 after retry RPC after error@116692927) -> (continuation of Deferred@1085840722 after retry RPC after error@170454536) -> (continuation of Deferred@1586759314 after retry RPC after error@2097101712) -> (continuation of Deferred@2015984464 after retry RPC after error@901546739) -> (continuation of Deferred@1264755264 after retry RPC after error@1313088263) -> (continuation of Deferred@2105815652 after retry RPC after error@746495037) -> (continuation of Deferred@1926257488 after retry RPC after error@493031958) -> (continuation of Deferred@1101530316 after retry RPC after error@426764208) -> (continuation of Deferred@1771386134 after retry RPC after error@1826847120) -> (continuation of Deferred@1835835044 after retry RPC after error@190359262) -> (continuation of Deferred@1260724006 after retry RPC after error@1014874155) -> (continuation of Deferred@904782014 after retry RPC after error@2068700811) -> (continuation of Deferred@418525931 after retry RPC after error@1247593798) -> (continuation of Deferred@284046499 after retry RPC after error@96602547) -> (continuation of Deferred@212735626 after retry RPC after error@544507052) -> (continuation of Deferred@1096738605 after retry RPC after error@1413804324), errback=(continuation of Deferred@608359062 after retry RPC after error@1090985375) -> (continuation of Deferred@146524969 after retry RPC after error@405527121) -> (continuation of Deferred@718237088 after retry RPC after error@1398630019) -> (continuation of Deferred@90770754 after retry RPC after error@1138145004) -> (continuation of Deferred@304439643 after retry RPC after error@720659682) -> (continuation of Deferred@1731887075 after retry RPC after error@288216126) -> (continuation of Deferred@683364601 after retry RPC after error@568455800) -> (continuation of Deferred@353387712 after retry RPC after error@1784938104) -> (continuation of Deferred@414013269 after retry RPC after error@913376266) -> (continuation of Deferred@558919744 after retry RPC after error@1421204012) -> (continuation of Deferred@642791440 after retry RPC after error@415181188) -> (continuation of Deferred@986679459 after retry RPC after error@1079066988) -> (continuation of Deferred@1743151566 after retry RPC after error@116692927) -> (continuation of Deferred@1085840722 after retry RPC after error@170454536) -> (continuation of Deferred@1586759314 after retry RPC after error@2097101712) -> (continuation of Deferred@2015984464 after retry RPC after error@901546739) -> (continuation of Deferred@1264755264 after retry RPC after error@1313088263) -> (continuation of Deferred@2105815652 after retry RPC after error@746495037) -> (continuation of Deferred@1926257488 after retry RPC after error@493031958) -> (continuation of Deferred@1101530316 after retry RPC after error@426764208) -> (continuation of Deferred@1771386134 after retry RPC after error@1826847120) -> (continuation of Deferred@1835835044 after retry RPC after error@190359262) -> (continuation of Deferred@1260724006 after retry RPC after error@1014874155) -> (continuation of Deferred@904782014 after retry RPC after error@2068700811) -> (continuation of Deferred@418525931 after retry RPC after error@1247593798) -> (continuation of Deferred@284046499 after retry RPC after error@96602547) -> (continuation of Deferred@212735626 after retry RPC after error@544507052) -> (continuation of Deferred@1096738605 after retry RPC after error@1413804324)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduTransaction.doCommitTransaction(KuduTransaction.java:491)
at org.apache.kudu.client.KuduTransaction.commitWithMode(KuduTransaction.java:537)
at org.apache.kudu.client.KuduTransaction.commit(KuduTransaction.java:284)
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveSwitchesToOtherTxnManager(TestKuduTransaction.java:1152)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.9.82.126:43671,127.9.82.124:38119,127.9.82.125:45345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.126:43671(127.9.82.126:43671): Connection refused: /127.9.82.126:43671,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.82.124:38119(127.9.82.124:38119): Connection refused: /127.9.82.124:38119
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.9.82.124:38119
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)